You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by iw...@apache.org on 2016/09/09 02:00:51 UTC
[1/3] incubator-hawq git commit: HAWQ-1037. Add get OS user in
hdfs_config when is not set
Repository: incubator-hawq
Updated Branches:
refs/heads/master 51cf8634d -> 85000f34d
HAWQ-1037. Add get OS user in hdfs_config when is not set
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/85000f34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/85000f34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/85000f34
Branch: refs/heads/master
Commit: 85000f34dc226d4b5f5a16232136b173206aeae9
Parents: 65b0e27
Author: Chunling Wang <wa...@126.com>
Authored: Thu Sep 8 09:56:37 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Fri Sep 9 10:00:14 2016 +0800
----------------------------------------------------------------------
src/test/feature/lib/hdfs_config.cpp | 19 ++++++++++++++++---
src/test/feature/lib/hdfs_config.h | 7 ++-----
2 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/85000f34/src/test/feature/lib/hdfs_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.cpp b/src/test/feature/lib/hdfs_config.cpp
index 8a8ed81..b09c2d4 100644
--- a/src/test/feature/lib/hdfs_config.cpp
+++ b/src/test/feature/lib/hdfs_config.cpp
@@ -2,10 +2,10 @@
#include <string>
#include <vector>
#include <unordered_set>
+#include <pwd.h>
#include "hdfs_config.h"
#include "command.h"
-#include "psql.h"
#include "xml_parser.h"
#include "string_util.h"
@@ -14,6 +14,19 @@ using std::string;
namespace hawq {
namespace test {
+HdfsConfig::HdfsConfig() {
+ std::string user = HAWQ_USER;
+ if(user.empty()) {
+ struct passwd *pw;
+ uid_t uid = geteuid();
+ pw = getpwuid(uid);
+ user.assign(pw->pw_name);
+ }
+ conn.reset(new hawq::test::PSQL(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, user, HAWQ_PASSWORD));
+ isLoadFromHawqConfigFile = false;
+ isLoadFromHdfsConfigFile = false;
+}
+
void HdfsConfig::runCommand(const string &command,
bool ishdfsuser,
string &result) {
@@ -115,7 +128,7 @@ bool HdfsConfig::LoadFromHdfsConfigFile() {
}
int HdfsConfig::isHA() {
- const hawq::test::PSQLQueryResult &result = psql.getQueryResult(
+ const hawq::test::PSQLQueryResult &result = conn->getQueryResult(
"SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) "
"FROM pg_filespace pgfs, pg_filespace_entry pgfse "
"WHERE pgfs.fsname = 'dfs_system' AND pgfse.fsefsoid=pgfs.oid ;");
@@ -173,7 +186,7 @@ string HdfsConfig::getHadoopHome() {
}
bool HdfsConfig::getNamenodeHost(string &namenodehost) {
- const hawq::test::PSQLQueryResult &result = psql.getQueryResult(
+ const hawq::test::PSQLQueryResult &result = conn->getQueryResult(
"SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) "
"FROM pg_filespace pgfs, pg_filespace_entry pgfse "
"WHERE pgfs.fsname = 'dfs_system' AND pgfse.fsefsoid=pgfs.oid ;");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/85000f34/src/test/feature/lib/hdfs_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.h b/src/test/feature/lib/hdfs_config.h
index 6b57588..c339c83 100644
--- a/src/test/feature/lib/hdfs_config.h
+++ b/src/test/feature/lib/hdfs_config.h
@@ -21,10 +21,7 @@ class HdfsConfig {
/**
* HdfsConfig constructor
*/
- HdfsConfig(): psql(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD) {
- isLoadFromHawqConfigFile = false;
- isLoadFromHdfsConfigFile = false;
- }
+ HdfsConfig();
/**
* HdfsConfig destructor
@@ -180,7 +177,7 @@ class HdfsConfig {
std::unique_ptr<XmlConfig> hdfsxmlconf;
bool isLoadFromHawqConfigFile;
bool isLoadFromHdfsConfigFile;
- hawq::test::PSQL psql;
+ std::unique_ptr<hawq::test::PSQL> conn;
};
} // namespace test
[3/3] incubator-hawq git commit: HAWQ-1037. Modify way to get HDFS
port in TestHawqRegister
Posted by iw...@apache.org.
HAWQ-1037. Modify way to get HDFS port in TestHawqRegister
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4b202a2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4b202a2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4b202a2b
Branch: refs/heads/master
Commit: 4b202a2baae5e3ecca0fc68f10a6d2e0f4cbecc9
Parents: 51cf863
Author: Chunling Wang <wa...@126.com>
Authored: Wed Aug 31 18:19:05 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Fri Sep 9 10:00:14 2016 +0800
----------------------------------------------------------------------
.../ManagementTool/test_hawq_register.cpp | 122 +++++++++++++------
1 file changed, 87 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b202a2b/src/test/feature/ManagementTool/test_hawq_register.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp b/src/test/feature/ManagementTool/test_hawq_register.cpp
index 0952bd5..2f4291b 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register.cpp
@@ -3,18 +3,36 @@
#include "lib/command.h"
#include "lib/sql_util.h"
#include "lib/string_util.h"
+#include "lib/hdfs_config.h"
#include "gtest/gtest.h"
using std::string;
using hawq::test::SQLUtility;
using hawq::test::Command;
+using hawq::test::HdfsConfig;
/* This test suite may consume more than 80 seconds. */
class TestHawqRegister : public ::testing::Test {
public:
TestHawqRegister() {}
~TestHawqRegister() {}
+ string getHdfsLocation() {
+ HdfsConfig hc;
+ string hostname = "";
+ int port = 0;
+ if (hc.isHA()) {
+ hc.getActiveNamenode(hostname, port);
+ }
+ else {
+ std::vector<std::string> hostList;
+ std::vector<int> portList;
+ hc.getNamenodes(hostList, portList);
+ hostname = hostList[0];
+ port = portList[0];
+ }
+ return hawq::test::stringFormat("hdfs://%s:%d", hostname.c_str(), port);
+ }
};
TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) {
@@ -23,12 +41,14 @@ TEST_F(TestHawqRegister, TestUsage1SingleHawqFile) {
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 3);
util.execute("insert into hawqregister values(1);");
@@ -42,12 +62,14 @@ TEST_F(TestHawqRegister, TestUsage1SingleHiveFile) {
string relativePath("/ManagementTool/test_hawq_register_hive.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hive.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hive.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hive.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hive.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("insert into hawqregister values(1);");
@@ -63,12 +85,14 @@ TEST_F(TestHawqRegister, TestDataTypes) {
string relativePath("/ManagementTool/test_hawq_register_data_types.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_data_types.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_data_types.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("drop table hawqregister;");
@@ -83,12 +107,14 @@ TEST_F(TestHawqRegister, TestAllNULL) {
string relativePath("/ManagementTool/test_hawq_register_null.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_data_types.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_data_types.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(a bool, b int2, c int2, d int4, e int8, f date, g float4, h float8, i varchar, j bytea, k char, l varchar) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_data_types.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_data_types.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 1);
util.execute("drop table hawqregister;");
@@ -102,24 +128,33 @@ TEST_F(TestHawqRegister, TestFiles) {
relativePath = "/ManagementTool/test_hawq_register_hive.paq";
string filePath2 = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -mkdir -p hdfs://localhost:8020/hawq_register_test/t/t"));
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath1 + " hdfs://localhost:8020/hawq_register_test/hawq1.paq"));
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath1 + " hdfs://localhost:8020/hawq_register_test/hawq2.paq"));
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath1 + " hdfs://localhost:8020/hawq_register_test/t/hawq.paq"));
-
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath2 + " hdfs://localhost:8020/hawq_register_test/hive1.paq"));
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath2 + " hdfs://localhost:8020/hawq_register_test/hive2.paq"));
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath2 + " hdfs://localhost:8020/hawq_register_test/t/hive.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -mkdir -p %s %s/hawq_register_test/t/t", filePath1.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq1.paq", filePath1.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hawq2.paq", filePath1.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hawq.paq", filePath1.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive1.paq", filePath2.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/hive2.paq", filePath2.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
+ cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test/t/hive.paq", filePath2.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_test hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 12);
util.execute("insert into hawqregister values(1);");
util.query("select * from hawqregister;", 13);
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm -r hdfs://localhost:8020/hawq_register_test"));
+ cmd = hawq::test::stringFormat("hadoop fs -rm -r %s/hawq_register_test", getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
@@ -129,15 +164,18 @@ TEST_F(TestHawqRegister, TestUsage1HashDistributedTable) {
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet) distributed by (i);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm hdfs://localhost:8020/hawq_register_hawq.paq"));
+ cmd = hawq::test::stringFormat("hadoop fs -rm -r %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
@@ -147,15 +185,18 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetFile) {
string relativePath("/ManagementTool/test_hawq_register_not_paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_test_not_paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_test_not_paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int) with (appendonly=true, orientation=parquet);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_test_not_paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_test_not_paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm hdfs://localhost:8020/hawq_register_test_not_paq"));
+ cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_test_not_paq", getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
@@ -165,15 +206,18 @@ TEST_F(TestHawqRegister, TestUsage1NotParquetTable) {
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm hdfs://localhost:8020/hawq_register_hawq.paq"));
+ cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
@@ -183,7 +227,8 @@ TEST_F(TestHawqRegister, TestUsage1FileNotExist) {
util.execute("create table hawqregister(i int);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f /hdfs://localhost:8020hawq_register_file_not_exist hawqregister"));
+ auto cmd = hawq::test::stringFormat("hawq register -d %s -f %shawq_register_file_not_exist hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
util.execute("drop table hawqregister;");
@@ -195,15 +240,18 @@ TEST_F(TestHawqRegister, TestUsage1NotHDFSPath) {
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("create table hawqregister(i int);");
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f /hawq_register_hawq.paq hawqregister"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq hawqregister", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(1, Command::getCommandStatus(cmd));
util.query("select * from hawqregister;", 0);
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -rm hdfs://localhost:8020/hawq_register_hawq.paq"));
+ cmd = hawq::test::stringFormat("hadoop fs -rm %s/hawq_register_hawq.paq", getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table hawqregister;");
}
@@ -212,10 +260,12 @@ TEST_F(TestHawqRegister, TestUsage1ParquetRandomly) {
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table if exists nt;");
util.execute("create table nt(i int) with (appendonly=true, orientation=parquet);");
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq nt"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq nt", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from nt;", 3);
util.execute("insert into nt values(1);");
util.query("select * from nt;", 4);
@@ -227,10 +277,12 @@ TEST_F(TestHawqRegister, TestUsage1ParquetRandomly2) {
string rootPath(util.getTestRootPath());
string relativePath("/ManagementTool/test_hawq_register_hawq.paq");
string filePath = rootPath + relativePath;
- EXPECT_EQ(0, Command::getCommandStatus("hadoop fs -put -f " + filePath + " hdfs://localhost:8020/hawq_register_hawq.paq"));
+ auto cmd = hawq::test::stringFormat("hadoop fs -put -f %s %s/hawq_register_hawq.paq", filePath.c_str(), getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.execute("drop table if exists nt;");
util.execute("create table nt(i int) with (appendonly=true, orientation=parquet) distributed randomly;");
- EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -f hdfs://localhost:8020/hawq_register_hawq.paq nt"));
+ cmd = hawq::test::stringFormat("hawq register -d %s -f %s/hawq_register_hawq.paq nt", HAWQ_DB, getHdfsLocation().c_str());
+ EXPECT_EQ(0, Command::getCommandStatus(cmd));
util.query("select * from nt;", 3);
util.execute("insert into nt values(1);");
util.query("select * from nt;", 4);
[2/3] incubator-hawq git commit: HAWQ-1037. Get HDFS namenode host
from HAWQ catalog tables instead of from HDFS configuration file
Posted by iw...@apache.org.
HAWQ-1037. Get HDFS namenode host from HAWQ catalog tables instead of from HDFS configuration file
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/65b0e27c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/65b0e27c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/65b0e27c
Branch: refs/heads/master
Commit: 65b0e27c93dad8fce9e84fb0c8c1eb03e761cfdd
Parents: 4b202a2
Author: Chunling Wang <wa...@126.com>
Authored: Wed Sep 7 10:40:40 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Fri Sep 9 10:00:14 2016 +0800
----------------------------------------------------------------------
.../feature/ManagementTool/test_hawq_register.cpp | 16 +++-------------
src/test/feature/lib/hdfs_config.cpp | 14 ++++++++++++++
src/test/feature/lib/hdfs_config.h | 7 +++++++
3 files changed, 24 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65b0e27c/src/test/feature/ManagementTool/test_hawq_register.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.cpp b/src/test/feature/ManagementTool/test_hawq_register.cpp
index 2f4291b..385f959 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register.cpp
@@ -19,19 +19,9 @@ class TestHawqRegister : public ::testing::Test {
~TestHawqRegister() {}
string getHdfsLocation() {
HdfsConfig hc;
- string hostname = "";
- int port = 0;
- if (hc.isHA()) {
- hc.getActiveNamenode(hostname, port);
- }
- else {
- std::vector<std::string> hostList;
- std::vector<int> portList;
- hc.getNamenodes(hostList, portList);
- hostname = hostList[0];
- port = portList[0];
- }
- return hawq::test::stringFormat("hdfs://%s:%d", hostname.c_str(), port);
+ string namenodehost = "";
+ EXPECT_EQ(true, hc.getNamenodeHost(namenodehost));
+ return hawq::test::stringFormat("hdfs://%s", namenodehost.c_str());
}
};
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65b0e27c/src/test/feature/lib/hdfs_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.cpp b/src/test/feature/lib/hdfs_config.cpp
index cded21e..8a8ed81 100644
--- a/src/test/feature/lib/hdfs_config.cpp
+++ b/src/test/feature/lib/hdfs_config.cpp
@@ -172,6 +172,20 @@ string HdfsConfig::getHadoopHome() {
return hadoopHome;
}
+bool HdfsConfig::getNamenodeHost(string &namenodehost) {
+ const hawq::test::PSQLQueryResult &result = psql.getQueryResult(
+ "SELECT substring(fselocation from length('hdfs:// ') for (position('/' in substring(fselocation from length('hdfs:// ')))-1)::int) "
+ "FROM pg_filespace pgfs, pg_filespace_entry pgfse "
+ "WHERE pgfs.fsname = 'dfs_system' AND pgfse.fsefsoid=pgfs.oid ;");
+ std::vector<std::vector<string>> table = result.getRows();
+ if (table.size() > 0) {
+ namenodehost = table[0][0];
+ return true;
+ }
+
+ return false;
+}
+
bool HdfsConfig::getActiveNamenode(string &activenamenode,
int &port) {
return getHANamenode("active", activenamenode, port);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/65b0e27c/src/test/feature/lib/hdfs_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.h b/src/test/feature/lib/hdfs_config.h
index 2b68db1..6b57588 100644
--- a/src/test/feature/lib/hdfs_config.h
+++ b/src/test/feature/lib/hdfs_config.h
@@ -56,6 +56,13 @@ class HdfsConfig {
std::string getHadoopHome();
/**
+ * get HDFS namenode's host ('hostname:port' for non-HA, 'servicename' for HA)
+ * @ param namenodehost, namenode host reference which will be set
+ * @ return true if getNamenodeHost succeeded
+ */
+ bool getNamenodeHost(std::string &namenodehost);
+
+ /**
* get HDFS active namenode's hostname and port information
* @param activenamenode, active namenode hostname reference which will be set
* @param port, active namenode port reference which will be set