You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2016/10/28 00:22:04 UTC
incubator-hawq git commit: HAWQ-1113. Fix bug and add test for hawq
register force mode.
Repository: incubator-hawq
Updated Branches:
refs/heads/master ced05d56f -> 6cefb5529
HAWQ-1113. Fix bug and add test for hawq register force mode.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6cefb552
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6cefb552
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6cefb552
Branch: refs/heads/master
Commit: 6cefb5529c536df99fb864105e31e083c37741ea
Parents: ced05d5
Author: hzhang2 <zh...@163.com>
Authored: Fri Oct 28 08:22:48 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Oct 28 08:22:48 2016 +0800
----------------------------------------------------------------------
.../feature/ManagementTool/test_hawq_register.h | 2 +-
.../test_hawq_register_usage2_case2.cpp | 14 ++++-
src/test/feature/ManagementTool/usage2case2/10 | Bin 0 -> 535 bytes
src/test/feature/ManagementTool/usage2case2/11 | Bin 0 -> 535 bytes
src/test/feature/ManagementTool/usage2case2/12 | Bin 0 -> 535 bytes
src/test/feature/ManagementTool/usage2case2/13 | Bin 0 -> 535 bytes
src/test/feature/ManagementTool/usage2case2/8 | Bin 0 -> 535 bytes
src/test/feature/ManagementTool/usage2case2/9 | Bin 0 -> 543 bytes
.../usage2case2/same_path_yaml_config_tpl.yml | 57 +++++++++++++++++++
tools/bin/hawqregister | 35 +++++++++---
10 files changed, 97 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register.h
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register.h b/src/test/feature/ManagementTool/test_hawq_register.h
index 4edf2aa..dce1c8c 100644
--- a/src/test/feature/ManagementTool/test_hawq_register.h
+++ b/src/test/feature/ManagementTool/test_hawq_register.h
@@ -94,7 +94,7 @@ class TestHawqRegister : public ::testing::Test {
void runYamlCaseTableNotExists(std::string casename, std::string ymlname, int expectederror, int checknum);
void runYamlCaseTableExists(std::string casename, std::string ymlname, int isexpectederror, int checknum);
- void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror, int rows, int checknum);
+ void runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror, int rows, int checknum, bool samepath);
private:
std::unique_ptr<hawq::test::PSQL> conn;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
index 492dbd3..ad28a0a 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case2.cpp
@@ -80,7 +80,7 @@ TEST_F(TestHawqRegister, TestUsage2Case2Expected) {
}
}
-void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200) {
+void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ymlname, int isexpectederror = 1, int rows = 50, int checknum = 200, bool samepathfile = false) {
SQLUtility util;
string test_root(util.getTestRootPath());
string t_yml_tpl(hawq::test::stringFormat("%s/ManagementTool/%s_tpl.yml", test_root.c_str(), ymlname.c_str()));
@@ -107,11 +107,17 @@ void TestHawqRegister::runYamlCaseForceMode(std::string casename, std::string ym
hc.getNamenodeHost(hdfs_prefix);
strs_src_dst["@PORT@"]= hdfs_prefix;
frep.replace(t_yml_tpl, t_yml, strs_src_dst);
+ if(samepathfile) {
+ for(int i=8; i<=13; i++){
+ string file(hawq::test::stringFormat("%s/ManagementTool/usage2case2/", test_root.c_str()));
+ file.append(std::to_string(i));
+ EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hdfs dfs -put %s /hawq_default/16385/%s/%s/", file.c_str(), strs_src_dst["@DATABASE_OID@"].c_str(),strs_src_dst["@TABLE_OID_OLD@"].c_str())));
+ }
+ }
//printf("%s\n", hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str()).c_str());
//sleep(60);
EXPECT_EQ(isexpectederror, Command::getCommandStatus(hawq::test::stringFormat("hawq register --force -d %s -c %s testhawqregister_%s.nt", HAWQ_DB, t_yml.c_str(), casename.c_str())));
util.query("select * from nt;", checknum);
-
EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml.c_str())));
util.execute("drop table t;");
util.execute("drop table nt;");
@@ -250,6 +256,10 @@ TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlConfig) {
runYamlCaseForceMode("testusage2case2normalyamlconfig", "usage2case2/normal_yaml_config", 0, 50, 150);
}
+TEST_F(TestHawqRegister, TestUsage2Case2SamePathYamlConfig) {
+ runYamlCaseForceMode("testusage2case2samepathyamlconfig", "usage2case2/same_path_yaml_config", 0, 50, 332, true);
+}
+
TEST_F(TestHawqRegister, TestUsage2Case2NormalYamlNoUpdateConfig) {
runYamlCaseForceMode("testusage2case2normalyamlnoupdateconfig", "usage2case2/normal_yaml_no_update_config", 0, 50, 100);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/10
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/10 b/src/test/feature/ManagementTool/usage2case2/10
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/10 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/11
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/11 b/src/test/feature/ManagementTool/usage2case2/11
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/11 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/12
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/12 b/src/test/feature/ManagementTool/usage2case2/12
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/12 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/13
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/13 b/src/test/feature/ManagementTool/usage2case2/13
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/13 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/8
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/8 b/src/test/feature/ManagementTool/usage2case2/8
new file mode 100644
index 0000000..1aceb3f
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/8 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/9
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/9 b/src/test/feature/ManagementTool/usage2case2/9
new file mode 100644
index 0000000..1155c33
Binary files /dev/null and b/src/test/feature/ManagementTool/usage2case2/9 differ
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
new file mode 100644
index 0000000..9bba0aa
--- /dev/null
+++ b/src/test/feature/ManagementTool/usage2case2/same_path_yaml_config_tpl.yml
@@ -0,0 +1,57 @@
+Bucketnum: 6
+DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
+ dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
+ (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33
+DFS_URL: hdfs://@PORT@
+Distribution_Policy: DISTRIBUTED BY (i)
+Encoding: UTF8
+FileFormat: Parquet
+Parquet_FileLocations:
+ Checksum: false
+ CompressionLevel: 0
+ CompressionType: null
+ EnableDictionary: false
+ Files:
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/2
+ size: 250
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/4
+ size: 250
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/5
+ size: 258
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/6
+ size: 270
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/3
+ size: 258
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/1
+ size: 228
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/2
+ size: 215
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/3
+ size: 215
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/4
+ size: 220
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/1
+ size: 254
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/6
+ size: 215
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/5
+ size: 210
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/9
+ size: 543
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/8
+ size: 535
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/10
+ size: 535
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/11
+ size: 535
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/12
+ size: 535
+ - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/13
+ size: 535
+ PageSize: 1048576
+ RowGroupSize: 8388608
+Parquet_Schema:
+- name: i
+ type: int4
+TableName: public.nt
+Version: 1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6cefb552/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 46a9a8f..492b151 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -574,8 +574,10 @@ class HawqRegister(object):
sys.exit(1)
self.firstsegno, self.tabledir = self._get_metadata()
- seg_list, existed_sizes = self._get_metadata_from_table()
- existed_files = [self.tabledir + seg for seg in seg_list]
+ if self.yml and self.mode == 'force':
+ existed_files, existed_sizes = self._get_files_in_hdfs(self.tabledir)
+ else:
+ existed_files, existed_sizes = self._get_files_in_hdfs(self.filepath)
# check if file numbers in hdfs is consistent with the record count of pg_aoseg.
hdfs_file_no_lst = [f.split('/')[-1] for f in existed_files]
for k in range(1, self.firstsegno - 1):
@@ -740,13 +742,28 @@ class HawqRegister(object):
catalog_lst = [str(i) for i in range(1, segno)]
new_catalog_lst = [str(i) for i in range(segno, len(self.files_update) + 1)]
exist_catalog_lst = []
+ tmp_files_update = [f for f in self.files_update]
+ tmp_sizes_update = [f for f in self.sizes_update]
+
+ tmp_tupcounts_update = [f for f in self.tupcounts_update]
+ tmp_eofuncompresseds_update = [f for f in self.eofuncompresseds_update]
+ tmp_varblockcounts_update = [f for f in self.varblockcounts_update]
+
for k, seg in enumerate(update_segno_lst):
if seg not in catalog_lst:
- self.files_same_path.append(self.files_update[k])
- self.sizes_same_path.append(self.sizes_update[k])
- self.tupcounts_same_path.append(self.tupcounts_update[k])
- self.eofuncompresseds_same_path.append(self.eofuncompresseds_update[k])
- self.varblockcounts_same_path.append(self.varblockcounts_update[k])
+
+ self.files_same_path.append(tmp_files_update[k])
+ self.sizes_same_path.append(tmp_sizes_update[k])
+ self.tupcounts_same_path.append(tmp_tupcounts_update[k])
+ self.eofuncompresseds_same_path.append(tmp_eofuncompresseds_update[k])
+ self.varblockcounts_same_path.append(tmp_varblockcounts_update[k])
+
+ self.files_update.remove(tmp_files_update[k])
+ self.sizes_update.remove(tmp_sizes_update[k])
+ self.tupcounts_update.remove(tmp_tupcounts_update[k])
+ self.eofuncompresseds_update.remove(tmp_eofuncompresseds_update[k])
+ self.varblockcounts_update.remove(tmp_varblockcounts_update[k])
+
if seg in new_catalog_lst:
exist_catalog_lst.append(seg)
for seg in update_segno_lst:
@@ -757,11 +774,13 @@ class HawqRegister(object):
while (str(segno) in exist_catalog_lst):
segno += 1
self.segnos_same_path.append(segno)
+ segno += 1
for k, f in enumerate(self.files_same_path):
self.srcfiles.append(f)
self.dstfiles.append(self.tabledir + str(self.segnos_same_path[k]))
+ segno = self.firstsegno + len(self.files_same_path)
for f in self.files_append:
self.srcfiles.append(f)
self.dstfiles.append(self.tabledir + str(segno))
@@ -823,7 +842,7 @@ class HawqRegister(object):
if len(same_path_eofs) > 0:
query += 'insert into pg_aoseg.%s values(%d, %d, %d, %d)' % (self.seg_name, same_path_segno_lst[0], same_path_eofs[0], same_path_tupcounts[0], same_path_eofuncompresseds[0])
k = 0
- for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:], same_path_tupcounts[1:], same_path_eofuncompresseds[1]):
+ for same_path_eof, same_path_tupcount, same_path_eofuncompressed in zip(same_path_eofs[1:], same_path_tupcounts[1:], same_path_eofuncompresseds[1:]):
query += ',(%d, %d, %d, %d)' % (same_path_segno_lst[k + 1], same_path_eof, same_path_tupcount, same_path_eofuncompressed)
k += 1
query += ';'