You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/11/24 13:14:51 UTC
incubator-eagle git commit: [EAGLE-799] Can not get hbase info by
service api when the master becomes not available
Repository: incubator-eagle
Updated Branches:
refs/heads/master 8042f9e44 -> de061d599
[EAGLE-799] Can not get hbase info by service api when the master becomes not available
Can not get hbase info by service api when the master becomes not available
Author: yupu <yu...@ebay.com>
Closes #680 from puyulu/EAGLE-799.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/de061d59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/de061d59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/de061d59
Branch: refs/heads/master
Commit: de061d599c35735abd6b9447d37949289827e1e3
Parents: 8042f9e
Author: yupu <yu...@ebay.com>
Authored: Thu Nov 24 21:14:43 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Thu Nov 24 21:14:43 2016 +0800
----------------------------------------------------------------------
.../extractor/hbase/HbaseTopologyEntityParser.java | 5 +++--
.../extractor/hdfs/HdfsTopologyEntityParser.java | 13 +++++++++++++
.../topology/extractor/mr/MRTopologyEntityParser.java | 9 +++++----
.../eagle/topology/storm/TopologyDataPersistBolt.java | 2 +-
.../topology/entity/HdfsServiceTopologyAPIEntity.java | 13 +++++++++++++
5 files changed, 35 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
index 77781a3..f3eac23 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
@@ -46,7 +46,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser {
private Boolean kerberosEnable = false;
private TopologyRackResolver rackResolver;
- public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) {
+ public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) {
this.site = site;
this.rackResolver = resolver;
this.hBaseConfiguration = HBaseConfiguration.create();
@@ -56,7 +56,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser {
this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes);
// kerberos authentication
if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null
- && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) {
+ && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) {
this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal);
this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab);
this.kerberosEnable = true;
@@ -103,6 +103,7 @@ public class HbaseTopologyEntityParser implements TopologyEntityParser {
}
double liveRatio = liveServers * 1d / (liveServers + deadServers);
result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp));
+ result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, 1d, site, timestamp));
return result;
} catch (RuntimeException e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
index 627584a..72dce80 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hdfs/HdfsTopologyEntityParser.java
@@ -72,6 +72,8 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
private static final String DATA_NODE_CAPACITY = "capacity";
private static final String DATA_NODE_ADMIN_STATE = "adminState";
private static final String DATA_NODE_FAILED_VOLUMN = "volfails";
+ private static final String DATA_NODE_VERSION = "version";
+ private static final String NAME_NODE_VERSION = "Version";
private static final String DATA_NODE_DECOMMISSIONED = "Decommissioned";
private static final String DATA_NODE_DECOMMISSIONED_STATE = "decommissioned";
@@ -115,9 +117,18 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
final String urlString = buildFSNamesystemURL(url);
final Map<String, JMXBean> jmxBeanMap = JMXQueryHelper.query(urlString);
final JMXBean bean = jmxBeanMap.get(JMX_FS_NAME_SYSTEM_BEAN_NAME);
+
+ final String nameNodeUrlString = buildNamenodeInfo(url);
+ final Map<String, JMXBean> nameNodeMap = JMXQueryHelper.query(nameNodeUrlString);
+ final JMXBean nameNodeBean = nameNodeMap.get(JMX_NAMENODE_INFO);
+
if (bean == null || bean.getPropertyMap() == null) {
throw new ServiceNotResponseException("Invalid JMX format, FSNamesystem bean is null!");
}
+ if (nameNodeBean == null || nameNodeBean.getPropertyMap() == null) {
+ throw new ServiceNotResponseException("Invalid JMX format, NameNode bean is null!");
+ }
+
final String hostname = (String) bean.getPropertyMap().get(HA_NAME);
HdfsServiceTopologyAPIEntity result = createHdfsServiceEntity(TopologyConstants.NAME_NODE_ROLE, hostname, updateTime);
final String state = (String) bean.getPropertyMap().get(HA_STATE);
@@ -128,6 +139,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
result.setUsedCapacityTB(Double.toString(capacityUsedGB / 1024));
final Integer blocksTotal = (Integer) bean.getPropertyMap().get(BLOCKS_TOTAL);
result.setNumBlocks(Integer.toString(blocksTotal));
+ result.setVersion((String) nameNodeBean.getPropertyMap().get(NAME_NODE_VERSION));
return result;
}
@@ -240,6 +252,7 @@ public class HdfsTopologyEntityParser implements TopologyEntityParser {
} else {
entity.setStatus(TopologyConstants.DATA_NODE_LIVE_STATUS);
}
+ entity.setVersion(String.valueOf(liveNode.get(DATA_NODE_VERSION)));
numLiveNodes++;
result.getSlaveNodes().add(entity);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
index 671d07b..52148b7 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/mr/MRTopologyEntityParser.java
@@ -46,7 +46,7 @@ import static org.apache.eagle.topology.TopologyConstants.*;
public class MRTopologyEntityParser implements TopologyEntityParser {
- private String [] rmUrls;
+ private String[] rmUrls;
private String historyServerUrl;
private String site;
private TopologyRackResolver rackResolver;
@@ -91,9 +91,10 @@ public class MRTopologyEntityParser implements TopologyEntityParser {
// reSelect url
}
}
- if (result.getMasterNodes().isEmpty()) {
- result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE, 0, site, timestamp));
- }
+
+ double value = result.getMasterNodes().isEmpty() ? 0 : result.getMasterNodes().size() * 1d / rmUrls.length;
+ result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.RESOURCE_MANAGER_ROLE, value, site, timestamp));
+
doCheckHistoryServer(timestamp, result);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
index e0ff3cd..afd3518 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/storm/TopologyDataPersistBolt.java
@@ -90,7 +90,7 @@ public class TopologyDataPersistBolt extends BaseRichBolt {
GenericServiceAPIResponseEntity<TopologyBaseAPIEntity> response = client.search().query(query).pageSize(Integer.MAX_VALUE).send();
if (response.isSuccess() && response.getObj() != null) {
for (TopologyBaseAPIEntity entity : response.getObj()) {
- if (!availableHostnames.contains(generateKey(entity))) {
+ if (availableHostnames != null && availableHostnames.size() > 0 && !availableHostnames.contains(generateKey(entity))) {
entitiesForDeletion.add(entity);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/de061d59/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
index baca97b..d5ee41e 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
@@ -43,6 +43,8 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
private long writtenTxidDiff;
@Column("g")
private long lastUpdateTime;
+ @Column("h")
+ private String version;
public long getLastUpdateTime() {
return lastUpdateTime;
@@ -106,5 +108,16 @@ public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
this.writtenTxidDiff = writtenTxidDiff;
valueChanged("writtenTxidDiff");
}
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ valueChanged("version");
+ }
+
+
}