incubator-hawq git commit: HAWQ-742. Add hawq config test common library.

[hu...@apache.org]

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 00f6074bd -> b6dcecda2


HAWQ-742. Add hawq config test common library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b6dcecda
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b6dcecda
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b6dcecda

Branch: refs/heads/master
Commit: b6dcecda2483ade6c9394a449455f6b17213eec9
Parents: 00f6074
Author: hzhang2 <zh...@163.com>
Authored: Tue May 10 15:14:45 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Wed May 11 09:11:23 2016 +0800

----------------------------------------------------------------------
 src/test/feature/lib/hawq-config.cpp | 170 +++++++++++++++++++
 src/test/feature/lib/hawq-config.h   |  44 +++++
 src/test/feature/lib/psql.h          |   5 +-
 src/test/feature/lib/string-util.cpp |  95 +++++++++++
 src/test/feature/lib/string-util.h   |  35 ++++
 src/test/feature/lib/xml-parser.cpp  | 266 ++++++++++++++++++++++++++++++
 src/test/feature/lib/xml-parser.h    |  63 +++++++
 7 files changed, 676 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/hawq-config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hawq-config.cpp b/src/test/feature/lib/hawq-config.cpp
new file mode 100644
index 0000000..d973328
--- /dev/null
+++ b/src/test/feature/lib/hawq-config.cpp
@@ -0,0 +1,170 @@
+#include "hawq-config.h"
+
+#include <fstream>
+#include <iostream>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "command.h"
+#include "psql.h"
+#include "string-util.h"
+#include "xml-parser.h"
+
+bool HawqConfig::LoadFromConfigFile() {
+  const char *env = getenv("GPHOME");
+  std::string confPath = env ? env : "";
+  if (!confPath.empty()) {
+    confPath.append("/etc/hawq-site.xml");
+  } else {
+    return false;
+  }
+
+  xmlconf.reset(new XmlConfig(confPath.c_str()));
+  xmlconf->parse();
+  return true;
+}
+
+bool HawqConfig::getMaster(std::string &hostname, int &port) {
+  bool ret = LoadFromConfigFile();
+  if(!ret){
+    return false;
+  }
+  hostname = xmlconf->getString("hawq_master_address_host");
+  port = xmlconf->getInt32("hawq_master_address_port");
+  return true;
+}
+
+void HawqConfig::getStandbyMaster(std::string &hostname, int &port) {
+  PSQLQueryResult result = psql.getQueryResult(
+      "select hostname, port from gp_segment_configuration where role ='s'");
+  std::vector<std::vector<std::string> > table = result.getRows();
+  if (table.size() > 0) {
+    hostname = table[0][0];
+    std::string portStr = table[0][1];
+    port = std::stoi(portStr);
+  }
+}
+
+void HawqConfig::getTotalSegments(std::vector<std::string> &hostname,
+    std::vector<int> &port) {
+  PSQLQueryResult result = psql.getQueryResult(
+      "select hostname, port from gp_segment_configuration where role ='p'");
+  std::vector<std::vector<std::string> > table = result.getRows();
+  for (int i = 0; i < table.size(); i++) {
+    hostname.push_back(table[i][0]);
+    std::string portStr = table[i][1];
+    port.push_back(std::stoi(portStr));
+  }
+}
+
+void HawqConfig::getSlaves(std::vector<std::string> &hostname) {
+
+  std::ifstream inFile;
+  char* GPHOME = getenv("GPHOME");
+  if (GPHOME == nullptr) {
+    return;
+  }
+  std::string slaveFile(GPHOME);
+  slaveFile.append("/etc/slaves");
+  inFile.open(slaveFile.c_str());
+  std::string line;
+  while (std::getline(inFile, line)) {
+    hostname.push_back(line);
+  }
+  inFile.close();
+}
+
+void HawqConfig::getUpSegments(std::vector<std::string> &hostname,
+    std::vector<int> &port) {
+  PSQLQueryResult result =
+      psql.getQueryResult(
+          "select hostname, port from gp_segment_configuration where role = 'p' and status = 'u'");
+  std::vector<std::vector<std::string> > table = result.getRows();
+
+  if (table.size() > 0) {
+    hostname.push_back(table[0][0]);
+    std::string portStr = table[0][1];
+    port.push_back(std::stoi(portStr));
+  }
+}
+
+void HawqConfig::getDownSegments(std::vector<std::string> &hostname,
+    std::vector<int> &port) {
+  PSQLQueryResult result =
+      psql.getQueryResult(
+          "select hostname, port from gp_segment_configuration where role = 'p' and status != 'u'");
+  std::vector<std::vector<std::string> > table = result.getRows();
+
+  if (table.size() > 0) {
+    hostname.push_back(table[0][0]);
+    std::string portStr = table[0][1];
+    port.push_back(std::stoi(portStr));
+  }
+}
+
+std::string HawqConfig::getGucValue(std::string gucName) {
+  std::string cmd = "hawq config -s ";
+  cmd.append(gucName);
+  Command c(cmd);
+  std::string result = c.run().getResultOutput();
+  std::string gucValue = "";
+  std::vector<std::string> lines = StringUtil::split(result, '\n');
+  // second line is value output.
+  if (lines.size() >= 2) {
+    std::string valueLine = lines[1];
+    int pos = valueLine.find_first_of(':');
+    std::string value = valueLine.substr(pos + 1);
+    gucValue = StringUtil::trim(value);
+  }
+  return gucValue;
+}
+
+std::string HawqConfig::setGucValue(std::string gucName, std::string gucValue) {
+  std::string cmd = "hawq config -c ";
+  cmd.append(gucName);
+  cmd.append(" -v ");
+  cmd.append(gucValue);
+  Command c(cmd);
+  std::string ret = c.run().getResultOutput();
+  return ret;
+}
+
+bool HawqConfig::isMasterMirrorSynchronized() {
+  PSQLQueryResult result = psql.getQueryResult(
+      "select summary_state from gp_master_mirroring");
+  if (result.getRows().size() > 0) {
+    std::string syncInfo = result.getData(0, 0);
+    syncInfo = StringUtil::trim(syncInfo);
+    if (syncInfo == "Synchronized") {
+      return true;
+    } else {
+      return false;
+    }
+  }
+  return false;
+}
+
+bool HawqConfig::isMultinodeMode() {
+  PSQLQueryResult result = psql.getQueryResult(
+      "select hostname from gp_segment_configuration");
+  std::vector<std::vector<std::string> > table = result.getRows();
+
+  std::set<std::string> hostnameMap;
+  for (int i = 0; i < table.size(); i++) {
+    std::string hostname2 = table[i][0];
+    if (hostname2 == "localhost") {
+      char hostnamestr[256];
+      gethostname(hostnamestr, 255);
+      hostname2.assign(hostnamestr);
+    }
+    if (hostnameMap.find(hostname2) == hostnameMap.end()) {
+      hostnameMap.insert(hostname2);
+    }
+  }
+  if (hostnameMap.size() <= 1) {
+    return false;
+  } else {
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/hawq-config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hawq-config.h b/src/test/feature/lib/hawq-config.h
new file mode 100644
index 0000000..a32cc0c
--- /dev/null
+++ b/src/test/feature/lib/hawq-config.h
@@ -0,0 +1,44 @@
+#ifndef SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_
+#define SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_
+
+#include "psql.h"
+#include "xml-parser.h"
+
+class HawqConfig {
+  public:
+    HawqConfig(const std::string& user = "gpadmin",
+        const std::string& password = "", const std::string& db = "postgres",
+        const std::string& host = "localhost", const std::string& port = "5432") :
+        psql(db, host, port, user, password) {
+      std::string masterHostname = "";
+      int masterPort = 0;
+      bool ret = getMaster(masterHostname, masterPort);
+      if (ret) {
+        std::string masterPortStr = std::to_string(masterPort);
+        psql.setHost(masterHostname);
+        psql.setPort(masterPortStr);
+      }
+    }
+    ~HawqConfig() {
+    }
+
+    bool LoadFromConfigFile();
+    bool getMaster(std::string &hostname, int &port);
+    void getStandbyMaster(std::string &hostname, int &port);
+    void getTotalSegments(std::vector<std::string> &hostname,
+        std::vector<int> &port);
+    void getSlaves(std::vector<std::string> &hostname);
+    void getUpSegments(std::vector<std::string> &hostname,
+        std::vector<int> &port);
+    void getDownSegments(std::vector<std::string> &hostname,
+        std::vector<int> &port);
+    std::string getGucValue(std::string gucName);
+    std::string setGucValue(std::string gucName, std::string gucValue);
+    bool isMasterMirrorSynchronized();
+    bool isMultinodeMode();
+  private:
+    std::unique_ptr<XmlConfig> xmlconf;
+    PSQL psql;
+};
+
+#endif /* SRC_TEST_FEATURE_LIB_HAWQ_CONFIG_H_ */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/psql.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/psql.h b/src/test/feature/lib/psql.h
index fb6d64c..75bee61 100644
--- a/src/test/feature/lib/psql.h
+++ b/src/test/feature/lib/psql.h
@@ -1,10 +1,11 @@
 #ifndef __PSQL_H__
 #define __PSQL_H__
 
-#include "libpq-fe.h"
+
 #include <vector>
-#include "command.h"
 
+#include "command.h"
+#include "libpq-fe.h"
 class PSQLQueryResult
 {
 public:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/string-util.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/string-util.cpp b/src/test/feature/lib/string-util.cpp
new file mode 100644
index 0000000..4474c80
--- /dev/null
+++ b/src/test/feature/lib/string-util.cpp
@@ -0,0 +1,95 @@
+#include "string-util.h"
+
+#include <algorithm>
+#include <cassert>
+#include <regex>
+#include <string>
+
+bool StringUtil::iequals(const std::string &str1, const std::string &str2) {
+  if (str1.size() != str2.size()) {
+    return false;
+  }
+  for (std::string::const_iterator c1 = str1.begin(), c2 = str2.begin();
+       c1 != str1.end(); ++c1, ++c2) {
+    if (tolower(*c1) != tolower(*c2)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+void StringUtil::replace(std::string *subject, const std::string &search,
+                         const std::string &replace) {
+  size_t pos = 0;
+  while ((pos = subject->find(search, pos)) != std::string::npos) {
+    subject->replace(pos, search.length(), replace);
+    pos += replace.length();
+  }
+}
+
+void StringUtil::toLower(std::string *str) {
+  assert(str != nullptr);
+
+  std::transform(str->begin(), str->end(), str->begin(), ::tolower);
+}
+
+std::string StringUtil::lower(const std::string &str) {
+  std::string result;
+
+  for (std::string::const_iterator iter = str.begin(); iter != str.end();
+       iter++) {
+    char c = tolower(*iter);
+    result.append(&c, sizeof(char));
+  }
+
+  return std::move(result);
+}
+
+std::string &StringUtil::trim(std::string &s) {  // NOLINT
+  if (s.empty()) {
+    return s;
+  }
+  s.erase(0, s.find_first_not_of(" "));
+  s.erase(s.find_last_not_of(" ") + 1);
+  return s;
+}
+
+std::string &StringUtil::trimNewLine(std::string &s) {  // NOLINT
+  s.erase(std::remove(s.begin(), s.end(), '\n'), s.end());
+  return s;
+}
+
+std::vector<std::string> StringUtil::split(const std::string &s,
+                                           char delimiter) {
+  std::vector<std::string> v;
+
+  std::string::size_type i = 0;
+  std::string::size_type j = s.find(delimiter);
+  if (j == std::string::npos) {
+    v.push_back(s);
+  }
+  while (j != std::string::npos) {
+    v.push_back(s.substr(i, j - i));
+    i = ++j;
+    j = s.find(delimiter, j);
+
+    if (j == std::string::npos) v.push_back(s.substr(i, s.length()));
+  }
+  return v;
+}
+
+std::string StringUtil::regexReplace(std::string *subject,
+                                     const std::string &pattern,
+                                     const std::string &replace) {
+  const std::regex regPattern(pattern);
+  return std::regex_replace(*subject, regPattern, replace);
+}
+
+bool StringUtil::StartWith(const std::string &str,
+                           const std::string &strStart) {
+  if (str.empty() || strStart.empty()) {
+    return false;
+  }
+  return str.compare(0, strStart.size(), strStart) == 0 ? true : false;
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/string-util.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/string-util.h b/src/test/feature/lib/string-util.h
new file mode 100644
index 0000000..f593a8c
--- /dev/null
+++ b/src/test/feature/lib/string-util.h
@@ -0,0 +1,35 @@
+#ifndef SRC_TEST_FEATURE_LIB_STRING_UTIL_H_
+#define SRC_TEST_FEATURE_LIB_STRING_UTIL_H_
+
+
+#include <iomanip>
+#include <sstream>
+#include <string>
+#include <vector>
+
+class StringUtil {
+ public:
+  StringUtil() {}
+  ~StringUtil() {}
+
+  static bool iequals(const std::string &str1, const std::string &str2);
+  static void replace(std::string *subject, const std::string &search,
+                      const std::string &replace);
+  static std::string regexReplace(std::string *subject,
+                                  const std::string &pattern,
+                                  const std::string &replace);
+  static void toLower(std::string *str);
+  static std::string lower(const std::string &str);
+  static std::string &trim(std::string &s);         // NOLINT
+  static std::string &trimNewLine(std::string &s);  // NOLINT
+  static std::vector<std::string> split(const std::string &s, char delimiter);
+  static bool StartWith(const std::string &str, const std::string &strStart);
+
+  template <typename T>
+  static std::string toStringWithPrecision(const T value, const int n) {
+    std::ostringstream out;
+    out << std::setiosflags(std::ios::fixed) << std::setprecision(n) << value;
+    return out.str();
+  }
+};
+#endif /* SRC_TEST_FEATURE_LIB_STRING_UTIL_H_ */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/xml-parser.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml-parser.cpp b/src/test/feature/lib/xml-parser.cpp
new file mode 100644
index 0000000..7d19721
--- /dev/null
+++ b/src/test/feature/lib/xml-parser.cpp
@@ -0,0 +1,266 @@
+#include "xml-parser.h"
+
+#include <limits>
+
+XmlConfig::XmlConfig(const char *p) :
+    path(p) {
+  parse();
+}
+
+void XmlConfig::parse() {
+  // the result document tree
+  xmlDocPtr doc;
+  LIBXML_TEST_VERSION kv
+  .clear();
+
+  if (access(path.c_str(), R_OK)) {
+    return;
+  }
+
+  // parse the file
+  doc = xmlReadFile(path.c_str(), nullptr, 0);
+  if (doc == nullptr) {
+    return;
+  }
+  try {
+    // printf("ffff3\n");
+    readConfigItems(doc);
+    // printf("ffff4\n");
+    xmlFreeDoc(doc);
+  } catch (...) {
+    xmlFreeDoc(doc);
+    // LOG_ERROR(ERRCODE_INTERNAL_ERROR, "libxml internal error");
+  }
+}
+
+void XmlConfig::readConfigItems(xmlDocPtr doc) {
+  xmlNodePtr root, curNode;
+  root = xmlDocGetRootElement(doc);
+  if (root == nullptr || strcmp((const char *) root->name, "configuration")) {
+    return ;
+  }
+
+  // for each property
+
+  for (curNode = root->children; curNode != nullptr; curNode = curNode->next) {
+    if (curNode->type != XML_ELEMENT_NODE) {
+      continue;
+    }
+
+    if (strcmp((const char *) curNode->name, "property")) {
+      return;
+    }
+
+    readConfigItem(curNode->children);
+  }
+}
+
+void XmlConfig::readConfigItem(xmlNodePtr root) {
+  std::string key, value, scope;
+  xmlNodePtr curNode;
+  bool hasName = false, hasValue = false, hasScope = false;
+  for (curNode = root; curNode != nullptr; curNode = curNode->next) {
+    if (curNode->type != XML_ELEMENT_NODE) {
+      continue;
+    }
+
+    if (!hasName && !strcmp((const char *) curNode->name, "name")) {
+      if (curNode->children != nullptr
+          && XML_TEXT_NODE == curNode->children->type) {
+        key = (const char *) curNode->children->content;
+        hasName = true;
+      }
+    } else if (!hasValue && !strcmp((const char *) curNode->name, "value")) {
+      if (curNode->children != nullptr
+          && XML_TEXT_NODE == curNode->children->type) {
+        value = (const char *) curNode->children->content;
+        hasValue = true;
+      }
+    } else {
+      continue;
+    }
+  }
+
+  if (hasName && hasValue) {
+    kv[key] = value;
+    return;
+  } else if (hasName && hasValue) {
+    return;
+  } else if (hasName && hasScope) {
+    return;
+  } else if (hasName) {
+    return;
+  }
+}
+
+const char *XmlConfig::getString(const char *key) {
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return "";
+  }
+
+  return it->second.c_str();
+}
+
+const char *XmlConfig::getString(const char *key, const char *def) {
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return def;
+  } else {
+    return it->second.c_str();
+  }
+}
+
+const char *XmlConfig::getString(const std::string &key) {
+  return getString(key.c_str());
+}
+
+const char *XmlConfig::getString(const std::string &key,
+    const std::string &def) {
+  return getString(key.c_str(), def.c_str());
+}
+
+int64_t XmlConfig::getInt64(const char *key) {
+  int64_t retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return 0;
+  }
+
+  retval = strToInt64(it->second.c_str());
+
+  return retval;
+}
+
+int64_t XmlConfig::getInt64(const char *key, int64_t def) {
+  int64_t retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return def;
+  }
+
+  retval = strToInt64(it->second.c_str());
+
+  return retval;
+}
+
+int32_t XmlConfig::getInt32(const char *key) {
+  int32_t retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return 0;
+  }
+
+  retval = strToInt32(it->second.c_str());
+
+  return retval;
+}
+
+int32_t XmlConfig::getInt32(const char *key, int32_t def) {
+  int32_t retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return def;
+  }
+
+  retval = strToInt32(it->second.c_str());
+
+  return retval;
+}
+
+double XmlConfig::getDouble(const char *key) {
+  double retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return 0.0;
+  }
+
+  retval = strToDouble(it->second.c_str());
+
+  return retval;
+}
+
+double XmlConfig::getDouble(const char *key, double def) {
+  double retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return def;
+  }
+
+  retval = strToDouble(it->second.c_str());
+
+  return retval;
+}
+
+bool XmlConfig::getBool(const char *key) {
+  bool retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+      return false;
+  }
+
+  retval = strToBool(it->second.c_str());
+
+  return retval;
+}
+
+bool XmlConfig::getBool(const char *key, bool def) {
+  bool retval;
+  XmlConfigMapIterator it = kv.find(key);
+
+  if (kv.end() == it) {
+    return def;
+  }
+
+  retval = strToBool(it->second.c_str());
+
+  return retval;
+}
+
+int64_t XmlConfig::strToInt64(const char *str) {
+  int64_t retval;
+  char *end = nullptr;
+
+  retval = strtoll(str, &end, 0);
+
+  return retval;
+}
+
+int32_t XmlConfig::strToInt32(const char *str) {
+  int32_t retval;
+  char *end = nullptr;
+  retval = strtoll(str, &end, 0);
+
+  return retval;
+}
+
+bool XmlConfig::strToBool(const char *str) {
+  bool retval = false;
+
+  if (!strcasecmp(str, "true") || !strcmp(str, "1")) {
+    retval = true;
+  } else if (!strcasecmp(str, "false") || !strcmp(str, "0")) {
+    retval = false;
+  } else {
+    return false;
+  }
+
+  return retval;
+}
+
+double XmlConfig::strToDouble(const char *str) {
+  double retval;
+  char *end = nullptr;
+  retval = strtod(str, &end);
+
+  return retval;
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b6dcecda/src/test/feature/lib/xml-parser.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml-parser.h b/src/test/feature/lib/xml-parser.h
new file mode 100644
index 0000000..b2e3497
--- /dev/null
+++ b/src/test/feature/lib/xml-parser.h
@@ -0,0 +1,63 @@
+#ifndef SRC_TEST_FEATURE_LIB_XML_PARSER_H_
+#define SRC_TEST_FEATURE_LIB_XML_PARSER_H_
+
+
+#include <string>
+#include <unordered_map>
+
+#include "libxml/parser.h"
+
+typedef std::unordered_map<std::string, std::string> XmlConfigMap;
+typedef std::unordered_map<std::string, std::string>::const_iterator
+    XmlConfigMapIterator;
+
+class XmlConfig {
+ public:
+  explicit XmlConfig(const char *p);
+
+  // parse the configuration file
+  void parse();
+
+  // @param key The key of the configuration item
+  // @ def The default value
+  // @ return The value of configuration item
+  const char *getString(const char *key);
+
+  const char *getString(const char *key, const char *def);
+
+  const char *getString(const std::string &key);
+
+  const char *getString(const std::string &key, const std::string &def);
+
+  int64_t getInt64(const char *key);
+
+  int64_t getInt64(const char *key, int64_t def);
+
+  int32_t getInt32(const char *key);
+
+  int32_t getInt32(const char *key, int32_t def);
+
+  double getDouble(const char *key);
+
+  double getDouble(const char *key, double def);
+
+  bool getBool(const char *key);
+
+  bool getBool(const char *key, bool def);
+
+  XmlConfigMap *getConfigMap() { return &kv; }
+
+ private:
+  std::string path;
+  XmlConfigMap kv;  // key2Value
+
+  void readConfigItems(xmlDocPtr doc);
+  void readConfigItem(xmlNodePtr root);
+  int64_t strToInt64(const char *str);
+  int32_t strToInt32(const char *str);
+  bool strToBool(const char *str);
+  double strToDouble(const char *str);
+};
+
+
+#endif /* SRC_TEST_FEATURE_LIB_XML_PARSER_H_ */