You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by iw...@apache.org on 2016/08/24 01:51:59 UTC
[3/3] incubator-hawq git commit: HAWQ-969. Add getting configuration
from HDFS and YARN
HAWQ-969. Add getting configuration from HDFS and YARN
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d1aafe68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d1aafe68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d1aafe68
Branch: refs/heads/master
Commit: d1aafe6855118e6bfcd841793b6040180ccc83b5
Parents: 8cc4a04
Author: Chunling Wang <wa...@126.com>
Authored: Mon Aug 1 14:59:17 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Wed Aug 24 09:50:40 2016 +0800
----------------------------------------------------------------------
src/test/feature/lib/hdfs_config.cpp | 284 +++++++++++++++++++++++++++++
src/test/feature/lib/hdfs_config.h | 172 +++++++++++++++++
src/test/feature/lib/xml_parser.cpp | 116 +++++++++++-
src/test/feature/lib/xml_parser.h | 19 ++
src/test/feature/lib/yarn_config.cpp | 267 +++++++++++++++++++++++++++
src/test/feature/lib/yarn_config.h | 153 ++++++++++++++++
src/test/feature/testlib/test_lib.cpp | 65 +++++++
7 files changed, 1066 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/hdfs_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.cpp b/src/test/feature/lib/hdfs_config.cpp
new file mode 100644
index 0000000..90b1032
--- /dev/null
+++ b/src/test/feature/lib/hdfs_config.cpp
@@ -0,0 +1,284 @@
+#include <fstream>
+#include <string>
+#include <vector>
+#include <unordered_set>
+
+#include "hdfs_config.h"
+#include "command.h"
+#include "psql.h"
+#include "xml_parser.h"
+#include "string_util.h"
+
+using std::string;
+
+namespace hawq {
+namespace test {
+
+string HdfsConfig::getHdfsUser() {
+ string cmd = "ps aux|grep hdfs.server|grep -v grep";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 1) {
+ return hawq::test::trim(hawq::test::split(lines[lines.size()-1], ' ')[0]);
+ }
+ return "hdfs";
+}
+
+bool HdfsConfig::LoadFromHawqConfigFile() {
+ const char *env = getenv("GPHOME");
+ string confPath = env ? env : "";
+ if (!confPath.empty()) {
+ confPath.append("/etc/hdfs-client.xml");
+ } else {
+ return false;
+ }
+
+ hawqxmlconf.reset(new XmlConfig(confPath));
+ hawqxmlconf->parse();
+ return true;
+}
+
+bool HdfsConfig::LoadFromHdfsConfigFile() {
+ string confPath=getHadoopHome();
+ if (confPath == "")
+ return false;
+ confPath.append("/etc/hadoop/hdfs-site.xml");
+ hdfsxmlconf.reset(new XmlConfig(confPath));
+ hdfsxmlconf->parse();
+ return true;
+}
+
+bool HdfsConfig::isHA() {
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+ string nameservice = hawqxmlconf->getString("dfs.nameservices");
+ if (nameservice.length() > 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool HdfsConfig::isKerbos() {
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+ string authentication = hawqxmlconf->getString("hadoop.security.authentication");
+ if (authentication == "kerberos") {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool HdfsConfig::isTruncate() {
+ string cmd = "hadoop fs -truncate";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 1) {
+ string valueLine = lines[0];
+ int find = valueLine.find("-truncate: Unknown command");
+ if (find < 0)
+ return true;
+ }
+ return false;
+}
+
+string HdfsConfig::getHadoopHome() {
+ string cmd = "ps -ef|grep hadoop";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ string hadoopHome = "";
+ auto lines = hawq::test::split(result, '\n');
+ for (size_t i=0; i<lines.size()-1; i++) {
+ string valueLine = lines[i];
+ int pos = valueLine.find("-Dhadoop.home.dir=");
+ if (pos >=0 ) {
+ string valueTmp = valueLine.substr(pos+18);
+ int valueEnd = valueTmp.find_first_of(" ");
+ string value = valueTmp.substr(0, valueEnd);
+ hadoopHome = hawq::test::trim(value);
+ return hadoopHome;
+ }
+ }
+ return hadoopHome;
+}
+
+bool HdfsConfig::getActiveNamenode(string &activenamenode,
+ int &port) {
+ return getHANamenode("active", activenamenode, port);
+}
+
+bool HdfsConfig::getStandbyNamenode(string &standbynamenode,
+ int &port) {
+ return getHANamenode("standby", standbynamenode, port);
+}
+
+bool HdfsConfig::getHANamenode(const string &namenodetype,
+ string &namenode,
+ int &port) {
+ if (!isHA())
+ return false;
+ string namenodeService = "";
+ string nameServiceValue = hawqxmlconf->getString("dfs.nameservices");
+ string haNamenodesName = "dfs.ha.namenodes.";
+ haNamenodesName.append(hawq::test::trim(nameServiceValue));
+ string haNamenodesValue = hawqxmlconf->getString(haNamenodesName);
+ auto haNamenodes = hawq::test::split(haNamenodesValue, ',');
+ for (size_t i = 0; i < haNamenodes.size(); i++) {
+ string haNamenode = hawq::test::trim(haNamenodes[i]);
+ string cmd = "sudo -u ";
+ cmd.append(getHdfsUser());
+ cmd.append(" hdfs haadmin -getServiceState ");
+ cmd.append(haNamenode);
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 1) {
+ string valueLine = lines[0];
+ if (valueLine == namenodetype) {
+ namenodeService = haNamenode;
+ break;
+ }
+ }
+ }
+ string rpcAddressName = "dfs.namenode.rpc-address.gphd-cluster.";
+ rpcAddressName.append(namenodeService);
+ string rpcAddressValue = hawqxmlconf->getString(rpcAddressName);
+ auto namenodeInfo = hawq::test::split(rpcAddressValue, ':');
+ namenode = hawq::test::trim(namenodeInfo[0]);
+ port = std::stoi(hawq::test::trim(namenodeInfo[1]));
+ return true;
+}
+
+void HdfsConfig::getNamenodes(std::vector<string> &namenodes,
+ std::vector<int> &port) {
+ string cmd = "sudo -u ";
+ cmd.append(getHdfsUser());
+ cmd.append(" hdfs getconf -nnRpcAddresses");
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ for (size_t i = 0; i < lines.size(); i++) {
+ string valueLine = lines[i];
+ auto namenodeInfo = hawq::test::split(valueLine, ':');
+ if (namenodeInfo.size() == 2) {
+ namenodes.push_back(hawq::test::trim(namenodeInfo[0]));
+ port.push_back(std::stoi(hawq::test::trim(namenodeInfo[1])));
+ }
+ }
+}
+
+void HdfsConfig::getDatanodelist(std::vector<string> &datanodelist,
+ std::vector<int> &port) {
+ string cmd = "sudo -u ";
+ cmd.append(getHdfsUser());
+ cmd.append(" hdfs dfsadmin -report | grep Name");
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ for (size_t i = 0; i < lines.size(); i++) {
+ string valueLine = lines[i];
+ auto datanodeInfo = hawq::test::split(valueLine, ':');
+ if (datanodeInfo.size() == 3) {
+ int portStart = datanodeInfo[2].find_first_of('(');
+ int portEnd = datanodeInfo[2].find_first_of(')');
+ string datanodePort = datanodeInfo[2].substr(0, portStart);
+ string datanodeHost = datanodeInfo[2].substr(portStart+1, portEnd-portStart-1);
+ datanodelist.push_back(hawq::test::trim(datanodeHost));
+ port.push_back(std::stoi(hawq::test::trim(datanodePort)));
+ }
+ }
+}
+
+void HdfsConfig::getActiveDatanodes(std::vector<string> &activedatanodes,
+ std::vector<int> &port) {
+ string cmd = "sudo -u ";
+ cmd.append(getHdfsUser());
+ cmd.append(" hdfs dfsadmin -report -live | grep Name");
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ for (size_t i = 0; i < lines.size(); i++) {
+ string valueLine = lines[i];
+ auto datanodeInfo = hawq::test::split(valueLine, ':');
+ if (datanodeInfo.size() == 3) {
+ int portStart = datanodeInfo[2].find_first_of('(');
+ int portEnd = datanodeInfo[2].find_first_of(')');
+ string datanodePort = datanodeInfo[2].substr(0, portStart);
+ string datanodeHost = datanodeInfo[2].substr(portStart+1, portEnd-portStart-1);
+ activedatanodes.push_back(hawq::test::trim(datanodeHost));
+ port.push_back(std::stoi(hawq::test::trim(datanodePort)));
+ }
+ }
+}
+
+
+bool HdfsConfig::isSafemode() {
+ string cmd = "hadoop fs -mkdir /tmp_hawq_test";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 1) {
+ string valueLine = lines[0];
+ int find = valueLine.find("Name node is in safe mode.");
+ if (find >= 0)
+ return true;
+ }
+ cmd = "hadoop fs -rm -r /tmp_hawq_test";
+ Command c_teardown(cmd);
+ result = c_teardown.run().getResultOutput();
+ return false;
+}
+
+string HdfsConfig::getParameterValue(const string ¶meterName) {
+ bool ret = LoadFromHdfsConfigFile();
+ if (!ret) {
+ return NULL;
+ }
+
+ return hdfsxmlconf->getString(parameterName);
+}
+
+string HdfsConfig::getParameterValue(const string ¶meterName,
+ const string &conftype) {
+ if (conftype == "hdfs" || conftype == "HDFS")
+ return getParameterValue(parameterName);
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return NULL;
+ }
+
+ return hawqxmlconf->getString(parameterName);
+}
+
+bool HdfsConfig::setParameterValue(const string ¶meterName,
+ const string ¶meterValue) {
+ bool ret = LoadFromHdfsConfigFile();
+ if (!ret) {
+ return false;
+ }
+
+ return hdfsxmlconf->setString(parameterName, parameterValue);
+}
+
+bool HdfsConfig::setParameterValue(const string ¶meterName,
+ const string ¶meterValue,
+ const string &conftype) {
+ if (conftype == "hdfs" || conftype == "HDFS")
+ return setParameterValue(parameterName, parameterValue);
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+
+ return hawqxmlconf->setString(parameterName, parameterValue);
+}
+
+} // namespace test
+} // namespace hawq
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/hdfs_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.h b/src/test/feature/lib/hdfs_config.h
new file mode 100644
index 0000000..2ea89f5
--- /dev/null
+++ b/src/test/feature/lib/hdfs_config.h
@@ -0,0 +1,172 @@
+#ifndef HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+#define HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+
+#include <string>
+#include <vector>
+
+#include "psql.h"
+#include "sql_util.h"
+#include "xml_parser.h"
+
+namespace hawq {
+namespace test {
+
+/**
+ * HdfsConfig common libray. Get detailed information about HDFS
+ * including checking state of namenodes and datanodes, get parameter value
+ * @author Chunling Wang
+ */
+class HdfsConfig {
+ public:
+ /**
+ * HdfsConfig constructor
+ */
+ HdfsConfig(): psql(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD) {}
+
+ /**
+ * HdfsConfig desstructor
+ */
+ ~HdfsConfig() {}
+
+ /**
+ * whether HDFS is in HA mode
+ * @return true if HDFS is HA
+ */
+ bool isHA();
+
+ /**
+ * whether HDFS is kerbos
+ * @return true if HDFS is kerbos
+ */
+ bool isKerbos();
+
+ /**
+ * whether HDFS supports truncate operation
+ * @return true if HDFS supports truncate operation
+ */
+ bool isTruncate();
+
+ /**
+ * get HADOOP working directory
+ * @return HADOOP working directory
+ */
+ std::string getHadoopHome();
+
+ /**
+ * get HDFS active namenode's hostname and port information
+ * @param activenamenode, active namenode hostname reference which will be set
+ * @param port, active namenode port reference which will be set
+ * @return true if getActiveNamenode succeeded
+ */
+ bool getActiveNamenode(std::string &activenamenode,
+ int &port);
+
+ /**
+ * get HDFS standby namenode's hostname and port information
+ * @param standbynamenode, standby namenode hostname reference which will be set
+ * @param port, standby namenode port reference which will be set
+ * @return true if getStandbyNamenode succeeded
+ */
+ bool getStandbyNamenode(std::string &standbynamenode,
+ int &port);
+
+ /**
+ * get HDFS namenode(s) information
+ * @param namenodes, namenodes' hostnames reference which will be set
+ * @param port, namenodes' ports reference which will be set
+ */
+ void getNamenodes(std::vector<std::string> &namenodes,
+ std::vector<int> &port);
+
+ /**
+ * get HDFS datanodes information
+ * @param datanodelist, datanodes' hostnames reference which will be set
+ * @param port, datanodes' ports reference which will be set
+ */
+ void getDatanodelist(std::vector<std::string> &datanodelist,
+ std::vector<int> &port);
+
+ /**
+ * get HDFS active datanodes information
+ * @param activedatanodes, active datanodes' hostnames reference which will be set
+ * @param port, active datanodes' ports reference which will be set
+ */
+ void getActiveDatanodes(std::vector<std::string> &activedatanodes,
+ std::vector<int> &port);
+
+ /**
+ * whether HDFS is in safe mode
+ * @return true if HDFS is in safe node
+ */
+ bool isSafemode();
+
+ /**
+ * get parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, used to get parameter value
+ * @param conftype, get parameter value, 'hdfs' or 'HDFS' from ./etc/hdfs-client.xml, others from ./etc/hadoop/hdfs-site.xml
+ * @return parameter value
+ */
+ std::string getParameterValue(const std::string ¶meterName, const std::string &conftype);
+
+ /**
+ * get parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, used to get parameter value
+ * @return parameter value
+ */
+ std::string getParameterValue(const std::string ¶meterName);
+
+ /**
+ * set parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, parameter name which used to set parameter value
+ * @param parameterValue, parameter value which to be set
+ * @param conftype, get parameter value, 'hdfs' or 'HDFS' from ./etc/hdfs-client.xml, others from ./etc/hadoop/hdfs-site.xml
+ * @return true if succeeded
+ */
+ bool setParameterValue(const std::string ¶meterName, const std::string ¶meterValue, const std::string &conftype);
+
+ /**
+ * set parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, parameter name which used to set parameter value
+ * @param parameterValue, parameter value which to be set
+ * @return true if succeeded
+ */
+ bool setParameterValue(const std::string ¶meterName, const std::string ¶meterValue);
+
+ private:
+ /**
+ * @return yarn user
+ */
+ std::string getHdfsUser();
+ /**
+ * load key-value parameters in ./etc/hdfs-client.xml
+ * @return true if succeeded
+ */
+ bool LoadFromHawqConfigFile();
+
+ /**
+ * load key-value parameters in ./etc/hadoop/hdfs-site.xml
+ * @return true if succeeded
+ */
+ bool LoadFromHdfsConfigFile();
+
+ /**
+ * get HDFS active or standby namenode information in HA mode according to the namenodetype
+ * @param namenodetype, used to specify active or standby namenode information
+ * @param namenode, namenode hostname reference which will be set
+ * @param port, namenode port reference which will be set
+ * @return true if getHANamenode succeeded
+ */
+ bool getHANamenode(const std::string &namenodetype,
+ std::string &namenode,
+ int &port);
+
+ private:
+ std::unique_ptr<XmlConfig> hawqxmlconf;
+ std::unique_ptr<XmlConfig> hdfsxmlconf;
+ hawq::test::PSQL psql;
+};
+
+} // namespace test
+} // namespace hawq
+
+#endif /* HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/xml_parser.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml_parser.cpp b/src/test/feature/lib/xml_parser.cpp
index bd1edd0..3cdc229 100644
--- a/src/test/feature/lib/xml_parser.cpp
+++ b/src/test/feature/lib/xml_parser.cpp
@@ -14,26 +14,40 @@ XmlConfig::XmlConfig(string p) :
parse();
}
-void XmlConfig::parse() {
- // the result document tree
- xmlDocPtr doc;
- LIBXML_TEST_VERSION kv
- .clear();
-
+bool XmlConfig::open() {
if (access(path.c_str(), R_OK)) {
- return;
+ return false;
}
- // parse the file
doc = xmlReadFile(path.c_str(), nullptr, 0);
if (doc == nullptr) {
+ return false;
+ }
+
+ return true;
+}
+
+void XmlConfig::closeNotSave() {
+ xmlFreeDoc(doc);
+}
+
+void XmlConfig::closeAndSave() {
+ xmlSaveFormatFile(path.c_str(), doc, 0);
+ xmlFreeDoc(doc);
+}
+
+void XmlConfig::parse() {
+ LIBXML_TEST_VERSION kv
+ .clear();
+
+ if (!open()) {
return;
}
try {
readConfigItems(doc);
- xmlFreeDoc(doc);
+ closeNotSave();
} catch (...) {
- xmlFreeDoc(doc);
+ closeNotSave();
}
}
@@ -91,6 +105,88 @@ void XmlConfig::readConfigItem(xmlNodePtr root) {
}
}
+bool XmlConfig::setString(const string &key,
+ const string &value,
+ const bool &save) {
+ bool result = false;
+
+ if (save) {
+ if (!open()) {
+ return false;
+ }
+ }
+ try {
+ result = writeConfigItem(doc, key, value);
+ if (save) {
+ closeAndSave();
+ }
+ } catch (...) {
+ if (save) {
+ closeNotSave();
+ }
+ }
+ return result;
+}
+
+bool XmlConfig::setString(const string &key,
+ const string &value) {
+ return setString(key, value, true);
+}
+
+bool XmlConfig::writeConfigItem(xmlDocPtr doc,
+ const string &key,
+ const string &value) {
+ xmlNodePtr root, curNode, curNodeChild, findNode;
+ root = xmlDocGetRootElement(doc);
+ bool findkey = false;
+ if (root == nullptr || strcmp((const char *) root->name, "configuration")) {
+ return false;
+ }
+ // for each property
+ for (curNode = root->children; curNode != nullptr; curNode = curNode->next) {
+ if (curNode->type != XML_ELEMENT_NODE) {
+ continue;
+ }
+ if (strcmp((const char *) curNode->name, "property")) {
+ return false;
+ }
+ curNodeChild = curNode->children;
+ for (curNodeChild = curNode->children; curNodeChild != nullptr; curNodeChild = curNodeChild->next) {
+ if (curNodeChild->type != XML_ELEMENT_NODE) {
+ continue;
+ }
+ if (!strcmp((const char *) curNodeChild->name, "name")) {
+ if (curNodeChild->children != nullptr
+ && XML_TEXT_NODE == curNodeChild->children->type
+ && !strcmp((const char *) curNodeChild->children->content, key.c_str())) {
+ findNode = curNode;
+ findkey = true;
+ } else {
+ if (findkey) {
+ xmlNewTextChild(findNode, NULL, BAD_CAST "value", BAD_CAST value.c_str());
+ return true;
+ }
+ }
+ } else if (!strcmp((const char *) curNodeChild->name, "value")) {
+ if (findkey
+ && curNodeChild->children != nullptr
+ && XML_TEXT_NODE == curNodeChild->children->type) {
+ xmlNodeSetContent(curNodeChild->children, (xmlChar*) const_cast<char*>(value.c_str()));
+ return true;
+ }
+ } else {
+ continue;
+ }
+ }
+ }
+
+ xmlNodePtr newNode = xmlNewNode(NULL, BAD_CAST "property");
+ xmlAddChild(root, newNode);
+ xmlNewTextChild(newNode, NULL, BAD_CAST "name", BAD_CAST key.c_str());
+ xmlNewTextChild(newNode, NULL, BAD_CAST "value", BAD_CAST value.c_str());
+ return true;
+}
+
const string XmlConfig::getString(const char *key) {
XmlConfigMapIterator it = kv.find(key);
if (kv.end() == it) {
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/xml_parser.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml_parser.h b/src/test/feature/lib/xml_parser.h
index 6978218..dde3b34 100644
--- a/src/test/feature/lib/xml_parser.h
+++ b/src/test/feature/lib/xml_parser.h
@@ -17,10 +17,27 @@ class XmlConfig {
public:
explicit XmlConfig(std::string);
+ // read an XML file into a tree
+ bool open();
+
+ // only free the XML document pointer
+ void closeNotSave();
+
+ // save the updated document to disk and free the XML document pointer
+ void closeAndSave();
+
// parse the configuration file
void parse();
// @param key The key of the configuration item
+ // @param value The updated value
+ // @param save whether save the updated document to disk, if save is false, open() and closeAndSave() should be called additionally
+ // @ return The value of configuration item
+ bool setString(const std::string &key, const std::string &value, const bool &save);
+
+ bool setString(const std::string &, const std::string &);
+
+ // @param key The key of the configuration item
// @ def The default value
// @ return The value of configuration item
const std::string getString(const char*);
@@ -52,6 +69,7 @@ class XmlConfig {
private:
void readConfigItems(xmlDocPtr doc);
void readConfigItem(xmlNodePtr root);
+ bool writeConfigItem(xmlDocPtr , const std::string &, const std::string &);
int64_t strToInt64(const char *);
int32_t strToInt32(const char *);
bool strToBool(const char *);
@@ -60,6 +78,7 @@ class XmlConfig {
private:
std::string path;
XmlConfigMap kv; // key2Value
+ xmlDocPtr doc;
}; // class XmlConfig
} // namespace test
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/yarn_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/yarn_config.cpp b/src/test/feature/lib/yarn_config.cpp
new file mode 100644
index 0000000..79211be
--- /dev/null
+++ b/src/test/feature/lib/yarn_config.cpp
@@ -0,0 +1,267 @@
+#include <fstream>
+#include <string>
+#include <vector>
+#include <unordered_set>
+
+#include "yarn_config.h"
+#include "command.h"
+#include "psql.h"
+#include "xml_parser.h"
+#include "string_util.h"
+
+using std::string;
+
+namespace hawq {
+namespace test {
+
+string YarnConfig::getYarnUser() {
+ string cmd = "ps aux|grep yarn.server|grep -v grep";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 1) {
+ return hawq::test::trim(hawq::test::split(lines[lines.size()-1], ' ')[0]);
+ }
+ return "yarn";
+}
+
+bool YarnConfig::LoadFromHawqConfigFile() {
+ const char *env = getenv("GPHOME");
+ string confPath = env ? env : "";
+ if (!confPath.empty()) {
+ confPath.append("/etc/yarn-client.xml");
+ } else {
+ return false;
+ }
+
+ hawqxmlconf.reset(new XmlConfig(confPath));
+ hawqxmlconf->parse();
+ return true;
+}
+
+bool YarnConfig::LoadFromYarnConfigFile() {
+ string confPath=getHadoopHome();
+ if (confPath == "")
+ return false;
+ confPath.append("/etc/hadoop/yarn-site.xml");
+ yarnxmlconf.reset(new XmlConfig(confPath));
+ yarnxmlconf->parse();
+ return true;
+}
+
+bool YarnConfig::isHA() {
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+ string nameservice = hawqxmlconf->getString("yarn.resourcemanager.ha");
+ if (nameservice.length() > 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool YarnConfig::isKerbos() {
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+ string authentication = hawqxmlconf->getString("hadoop.security.authentication");
+ if (authentication == "kerberos") {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+string YarnConfig::getHadoopHome() {
+ string cmd = "ps -ef|grep hadoop";
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ string hadoopHome = "";
+ auto lines = hawq::test::split(result, '\n');
+ for (size_t i=0; i<lines.size()-1; i++) {
+ string valueLine = lines[i];
+ int pos = valueLine.find("-Dhadoop.home.dir=");
+ if (pos >=0 ) {
+ string valueTmp = valueLine.substr(pos+18);
+ int valueEnd = valueTmp.find_first_of(" ");
+ string value = valueTmp.substr(0, valueEnd);
+ hadoopHome = hawq::test::trim(value);
+ return hadoopHome;
+ }
+ }
+ return hadoopHome;
+}
+
+bool YarnConfig::getActiveRM(string &activeRM,
+ int &port) {
+ return getHARM("active", activeRM, port);
+}
+
+bool YarnConfig::getStandbyRM(string &standbyRM,
+ int &port) {
+ return getHARM("standby", standbyRM, port);
+}
+
+bool YarnConfig::getHARM(const string &RMtype,
+ string &RM,
+ int &port) {
+ if (!isHA())
+ return false;
+ string RMService = "";
+ string haRMValue = "rm1,rm2";
+ auto haRMs = hawq::test::split(haRMValue, ',');
+ for (size_t i = 0; i < haRMs.size(); i++) {
+ string haRM = hawq::test::trim(haRMs[i]);
+ string cmd ="sudo -u ";
+ cmd.append(getYarnUser());
+ cmd.append(" yarn rmadmin -getServiceState ");
+ cmd.append(haRM);
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ if (lines.size() >= 2) {
+ string valueLine = lines[1];
+ if (valueLine == RMtype) {
+ RMService = haRM;
+ break;
+ }
+ }
+ }
+ bool ret = LoadFromYarnConfigFile();
+ if (!ret) {
+ return false;
+ }
+ string rpcAddressName = "yarn.resourcemanager.address.";
+ rpcAddressName.append(RMService);
+ string rpcAddressValue = yarnxmlconf->getString(rpcAddressName);
+ auto RMInfo = hawq::test::split(rpcAddressValue, ':');
+ RM = hawq::test::trim(RMInfo[0]);
+ port = std::stoi(hawq::test::trim(RMInfo[1]));
+ return true;
+}
+
+bool YarnConfig::getRMList(std::vector<string> &RMList,
+ std::vector<int> &ports){
+ string RM = "";
+ int port;
+ if (isHA()) {
+ getActiveRM(RM, port);
+ RMList.push_back(RM);
+ ports.push_back(port);
+ getStandbyRM(RM, port);
+ RMList.push_back(RM);
+ ports.push_back(port);
+ return true;
+ }
+
+ bool ret = LoadFromYarnConfigFile();
+ if (!ret) {
+ return false;
+ }
+
+ string RMAddressName = "yarn.resourcemanager.address";
+ string RMAddressValue = yarnxmlconf->getString(RMAddressName);
+ if (RMAddressValue == "")
+ return false;
+ auto RMInfo = hawq::test::split(RMAddressValue, ':');
+ RM = hawq::test::trim(RMInfo[0]);
+ port = std::stoi(hawq::test::trim(RMInfo[1]));
+ RMList.push_back(RM);
+ ports.push_back(port);
+ return true;
+}
+
+void YarnConfig::getNodeManagers(std::vector<string> &nodemanagers,
+ std::vector<int> &port) {
+ string cmd = "sudo -u ";
+ cmd.append(getYarnUser());
+ cmd.append(" yarn node -list -all");
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ bool begin = false;
+ for (size_t i=0; i<lines.size(); i++) {
+ if (!begin) {
+ if (lines[i].find("Node-Id") != string::npos) {
+ begin = true;
+ }
+ } else {
+ string values = hawq::test::split(lines[i], '\t')[0];
+ nodemanagers.push_back(hawq::test::trim(hawq::test::split(values, ':')[0]));
+ port.push_back(std::stoi(hawq::test::trim(hawq::test::split(values, ':')[1])));
+ }
+ }
+}
+
+void YarnConfig::getActiveNodeManagers(std::vector<string> &nodemanagers,
+ std::vector<int> &port) {
+ string cmd = "sudo -u ";
+ cmd.append(getYarnUser());
+ cmd.append(" yarn node -list -states RUNNING");
+ Command c(cmd);
+ string result = c.run().getResultOutput();
+ auto lines = hawq::test::split(result, '\n');
+ bool begin = false;
+ for (size_t i=0; i<lines.size(); i++) {
+ if (!begin) {
+ if (lines[i].find("Node-Id") != string::npos) {
+ begin = true;
+ }
+ } else {
+ string values = hawq::test::split(lines[i], '\t')[0];
+ nodemanagers.push_back(hawq::test::trim(hawq::test::split(values, ':')[0]));
+ port.push_back(std::stoi(hawq::test::trim(hawq::test::split(values, ':')[1])));
+ }
+ }
+}
+
+
+string YarnConfig::getParameterValue(const string ¶meterName) {
+ bool ret = LoadFromYarnConfigFile();
+ if (!ret) {
+ return NULL;
+ }
+
+ return yarnxmlconf->getString(parameterName);
+}
+
+string YarnConfig::getParameterValue(const string ¶meterName,
+ const string &conftype) {
+ if (conftype == "yarn" || conftype == "YARN")
+ return getParameterValue(parameterName);
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return NULL;
+ }
+
+ return hawqxmlconf->getString(parameterName);
+}
+
+bool YarnConfig::setParameterValue(const string ¶meterName,
+ const string ¶meterValue) {
+ bool ret = LoadFromYarnConfigFile();
+ if (!ret) {
+ return false;
+ }
+
+ return yarnxmlconf->setString(parameterName, parameterValue);
+}
+
+bool YarnConfig::setParameterValue(const string ¶meterName,
+ const string ¶meterValue,
+ const string &conftype) {
+ if (conftype == "yarn" || conftype == "YARN")
+ return setParameterValue(parameterName, parameterValue);
+ bool ret = LoadFromHawqConfigFile();
+ if (!ret) {
+ return false;
+ }
+
+ return hawqxmlconf->setString(parameterName, parameterValue);
+}
+
+} // namespace test
+} // namespace hawq
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/yarn_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/yarn_config.h b/src/test/feature/lib/yarn_config.h
new file mode 100644
index 0000000..559daca
--- /dev/null
+++ b/src/test/feature/lib/yarn_config.h
@@ -0,0 +1,153 @@
+#ifndef HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_
+#define HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_
+
+#include <string>
+#include <vector>
+
+#include "psql.h"
+#include "sql_util.h"
+#include "xml_parser.h"
+
+namespace hawq {
+namespace test {
+
+/**
+ * YarnConfig common libray. Get detailed information about YARN
+ * including checking state of resourcemanagers and nodemanagers, get parameter value
+ * @author Chunling Wang
+ */
+class YarnConfig {
+ public:
+ /**
+ * YarnConfig constructor
+ */
+ YarnConfig(): psql(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD) {}
+
+ /**
+ * YarnConfig desstructor
+ */
+ ~YarnConfig() {}
+
+ /**
+ * whether YARN is in HA mode
+ * @return true if YARN is HA
+ */
+ bool isHA();
+
+ /**
+ * whether YARN is kerbos
+ * @return true if YARN is kerbos
+ */
+ bool isKerbos();
+
+ /**
+ * get HADOOP working directory
+ * @return HADOOP working directory
+ */
+ std::string getHadoopHome();
+
+ /**
+ * get YARN active resourcemanager's hostname and port information
+ * @param activeresourcemanager, active resourcemanager hostname reference which will be set
+ * @param port, active resourcemanager port reference which will be set
+ * @return true if getActiveRM succeeded
+ */
+ bool getActiveRM(std::string &activeRM, int &port);
+
+ /**
+ * get YARN standby resourcemanager's hostname and port information
+ * @param standbyRM, standby resourcemanager hostname reference which will be set
+ * @param port, standby resourcemanager port reference which will be set
+ * @return true if getStandbyRM succeeded
+ */
+ bool getStandbyRM(std::string &standbyRM, int &port);
+
+ /**
+ * get YARN resourcemanager(s) information
+ * @param RMList, resourcemanagers' hostnames reference which will be set
+ * @param port, resourcemanagers' ports reference which will be set
+ */
+ bool getRMList(std::vector<std::string> &RMList, std::vector<int> &port);
+
+ /**
+ * get YARN nodemanagers information
+ * @param nodemanagers, nodemanagers' hostnames reference which will be set
+ * @param port, nodemanagers' ports reference which will be set
+ */
+ void getNodeManagers(std::vector<std::string> &nodemanagers, std::vector<int> &port);
+
+ /**
+ * get YARN active nodemanagers information
+ * @param nodemanagers, active nodemanagers' hostnames reference which will be set
+ * @param port, active nodemanagers' ports reference which will be set
+ */
+ void getActiveNodeManagers(std::vector<std::string> &nodemanagers, std::vector<int> &port);
+
+ /**
+ * get parameter value in ./etc/yarn-client.xml or ./etc/hadoop/yarn-site.xml according to parameter name
+ * @param parameterName, used to get parameter value
+ * @param conftype, get parameter value, 'yarn' or 'YARN' from ./etc/yarn-client.xml, others from ./etc/hadoop/yarn-site.xml
+ * @return parameter value
+ */
+ std::string getParameterValue(const std::string ¶meterName);
+
+ /**
+ * get parameter value in ./etc/hadoop/yarn-site.xml according to parameter name
+ * @param parameterName, used to get parameter value
+ * @return parameter value
+ */
+ std::string getParameterValue(const std::string ¶meterName, const std::string &conftype);
+
+ /**
+ * set parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, parameter name which used to set parameter value
+ * @param parameterValue, parameter value which to be set
+ * @param conftype, get parameter value, 'yarn' or 'YARN' from ./etc/yarn-client.xml, others from ./etc/hadoop/yarn-site.xml
+ * @return true if succeeded
+ */
+ bool setParameterValue(const std::string ¶meterName, const std::string ¶meterValue);
+
+ /**
+ * set parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+ * @param parameterName, parameter name which used to set parameter value
+ * @param parameterValue, parameter value which to be set
+ * @return true if succeeded
+ */
+ bool setParameterValue(const std::string ¶meterName, const std::string ¶meterValue, const std::string &conftype);
+
+ private:
+ /**
+ * @return yarn user
+ */
+ std::string getYarnUser();
+ /**
+ * load key-value parameters in ./etc/yarn-client.xml
+ * @return true if succeeded
+ */
+ bool LoadFromHawqConfigFile();
+
+ /**
+ * load key-value parameters in ./etc/hadoop/yarn-site.xml
+ * @return true if succeeded
+ */
+ bool LoadFromYarnConfigFile();
+
+ /**
+ * get Yarn active or standby resourcemanager information in HA mode according to the RMtype
+ * @param RMtype, used to specify active or standby resourcemanager information
+ * @param RM, resourcemanager hostname reference which will be set
+ * @param port, resourcemanager port reference which will be set
+ * @return true if getHARM succeeded
+ */
+ bool getHARM(const std::string &RMtype, std::string &RMnode, int &port);
+
+ private:
+ std::unique_ptr<XmlConfig> hawqxmlconf;
+ std::unique_ptr<XmlConfig> yarnxmlconf;
+ hawq::test::PSQL psql;
+};
+
+} // namespace test
+} // namespace hawq
+
+#endif /* HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/testlib/test_lib.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/testlib/test_lib.cpp b/src/test/feature/testlib/test_lib.cpp
index 98b90d2..a577c93 100644
--- a/src/test/feature/testlib/test_lib.cpp
+++ b/src/test/feature/testlib/test_lib.cpp
@@ -9,6 +9,8 @@
#include "lib/command.h"
#include "lib/data_gen.h"
#include "lib/hawq_config.h"
+#include "lib/hdfs_config.h"
+#include "lib/yarn_config.h"
#include "lib/sql_util.h"
#include "lib/string_util.h"
#include "lib/file_replace.h"
@@ -56,6 +58,69 @@ TEST_F(TestCommonLib, TestHawqConfig) {
return;
}
+TEST_F(TestCommonLib, TestHdfsConfig) {
+ hawq::test::HdfsConfig hc;
+ hc.isHA();
+ hc.isKerbos();
+ hc.isTruncate();
+ std::string hadoopHome = hc.getHadoopHome();
+
+ std::string hostname = "";
+ int port = 0;
+ hc.getActiveNamenode(hostname, port);
+
+ hostname = "";
+ port = 0;
+ hc.getStandbyNamenode(hostname, port);
+
+ std::vector<std::string> hostList;
+ std::vector<int> portList;
+ hc.getNamenodes(hostList, portList);
+
+ hostList.clear();
+ portList.clear();
+ hc.getDatanodelist(hostList, portList);
+
+ hostList.clear();
+ portList.clear();
+ hc.getActiveDatanodes(hostList, portList);
+
+ hc.isSafemode();
+
+ hc.getParameterValue("dfs.replication");
+ hc.setParameterValue("dfs.replication", "1");
+}
+
+TEST_F(TestCommonLib, TestYarnConfig) {
+ hawq::test::YarnConfig hc;
+ hc.isHA();
+ hc.isKerbos();
+ std::string hadoopHome = hc.getHadoopHome();
+
+ std::string hostname = "";
+ int port = 0;
+ hc.getActiveRM(hostname, port);
+
+ hostname = "";
+ port = 0;
+ hc.getStandbyRM(hostname, port);
+
+ std::vector<std::string> hostList;
+ std::vector<int> portList;
+ hc.getRMList(hostList, portList);
+
+ hostList.clear();
+ portList.clear();
+ hc.getNodeManagers(hostList, portList);
+
+ hostList.clear();
+ portList.clear();
+ hc.getActiveNodeManagers(hostList, portList);
+
+ hc.getParameterValue("yarn.scheduler.minimum-allocation-mb");
+ hc.setParameterValue("yarn.scheduler.minimum-allocation-mb", "1024");
+}
+
TEST_F(TestCommonLib, TestCommand) {
hawq::test::Command c("ls ./");
c.run().getResultOutput();