You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2016/10/31 06:18:51 UTC
incubator-hawq git commit: HAWQ-1034. Delete code for hawq register
repair mode
Repository: incubator-hawq
Updated Branches:
refs/heads/master 45adcbdb3 -> 778ab999a
HAWQ-1034. Delete code for hawq register repair mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/778ab999
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/778ab999
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/778ab999
Branch: refs/heads/master
Commit: 778ab999a95c36999430fea6ea29e7839106d9d1
Parents: 45adcbd
Author: Chunling Wang <wa...@126.com>
Authored: Mon Oct 31 13:50:14 2016 +0800
Committer: Chunling Wang <wa...@126.com>
Committed: Mon Oct 31 13:50:14 2016 +0800
----------------------------------------------------------------------
.../test_hawq_register_usage2_case3.cpp | 81 --------------------
.../ManagementTool/usage2case3/t_tpl_old_1.yml | 31 --------
.../ManagementTool/usage2case3/t_tpl_old_2.yml | 33 --------
.../ManagementTool/usage2case3/t_tpl_old_3.yml | 21 -----
.../ManagementTool/usage2case3/t_tpl_old_4.yml | 23 ------
tools/bin/hawqregister | 63 +++------------
6 files changed, 12 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
deleted file mode 100644
index 2b4d07b..0000000
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case3.cpp
+++ /dev/null
@@ -1,81 +0,0 @@
-#include <string>
-
-#include "lib/command.h"
-#include "lib/sql_util.h"
-#include "lib/string_util.h"
-#include "lib/hdfs_config.h"
-#include "lib/file_replace.h"
-#include "test_hawq_register.h"
-
-#include "gtest/gtest.h"
-
-using std::string;
-using hawq::test::SQLUtility;
-using hawq::test::Command;
-using hawq::test::HdfsConfig;
-
-TEST_F(TestHawqRegister, DISABLED_TestUsage2Case3Expected) {
- SQLUtility util;
- string test_root(util.getTestRootPath());
- std::vector<string> create_table_matrix = {"distributed by (i)", "distributed randomly"};
- std::vector<string> fmt_matrix = {"row", "parquet"};
- std::vector<string> option_matrix = {"--repair", "-R"};
- int suffix=0;
-
- for (auto & opt : option_matrix) {
- suffix = 0;
- for (auto & ddl : create_table_matrix) {
- for (auto & fmt : fmt_matrix) {
- suffix++;
- string t_yml_tpl_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_tpl_old_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
- string t_yml_old(hawq::test::stringFormat("%s/ManagementTool/usage2case3/t_old_%s.yml", test_root.c_str(), std::to_string(suffix).c_str()));
- auto t = hawq::test::stringFormat("t_usage2_case3_%s", std::to_string(suffix).c_str());
- auto nt = hawq::test::stringFormat("nt_usage2_case3_%s", std::to_string(suffix).c_str());
- util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
- util.execute(hawq::test::stringFormat("drop table if exists %s;", nt.c_str()));
-
- // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#, where nt_usage2_case3_# does not exist
- util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
- util.execute(hawq::test::stringFormat("insert into %s select generate_series(1, 100);", t.c_str()));
- util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 200);", t.c_str()));
- util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 200);
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str())));
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str())));
- util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 200);
-
- // hawq register -d hawq_feature_test -c t_usage2_case3_#.yml nt_usage2_case3_#, where nt_usage2_case3_# exists
- util.execute(hawq::test::stringFormat("drop table if exists %s;", t.c_str()));
- util.execute(hawq::test::stringFormat("create table %s(i int) with (appendonly=true, orientation=%s) %s;", t.c_str(), fmt.c_str(), ddl.c_str()));
- util.execute(hawq::test::stringFormat("insert into %s select generate_series(101, 150);", t.c_str()));
- util.query(hawq::test::stringFormat("select * from %s;", t.c_str()), 50);
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq extract -d %s -o t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), t.c_str())));
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register -d %s -c t_%s.yml testhawqregister_testusage2case3expected.%s", HAWQ_DB, std::to_string(suffix).c_str(), nt.c_str())));
- util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 250);
-
- // hawq register --repair/-R -d hawq_feature_test -c t_old_#.yml nt_usage2_case3_#
- hawq::test::FileReplace frep;
- std::unordered_map<std::string, std::string> strs_src_dst;
- strs_src_dst["@DATABASE_OID@"]= getDatabaseOid();
- strs_src_dst["@TABLE_OID@"]= getTableOid(nt);
- string hdfs_prefix;
- hawq::test::HdfsConfig hc;
- hc.getNamenodeHost(hdfs_prefix);
- strs_src_dst["@PORT@"]= hdfs_prefix;
- frep.replace(t_yml_tpl_old, t_yml_old, strs_src_dst);
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("hawq register %s -d %s -c %s testhawqregister_testusage2case3expected.%s", opt.c_str(), HAWQ_DB, t_yml_old.c_str(), nt.c_str())));
- util.query(hawq::test::stringFormat("select * from %s;", nt.c_str()), 100);
-
- if (fmt == "row")
- checkPgAOSegValue(nt, "-1", "aoseg");
- else
- checkPgAOSegValue(nt, "-1", "paqseg");
-
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf t_%s.yml", std::to_string(suffix).c_str())));
- EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf %s", t_yml_old.c_str())));
- util.execute(hawq::test::stringFormat("drop table %s;", t.c_str()));
- util.execute(hawq::test::stringFormat("drop table %s;", nt.c_str()));
- }
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
deleted file mode 100644
index 3ec2f4e..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_1.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-AO_FileLocations:
- Blocksize: 32768
- Checksum: false
- CompressionLevel: 0
- CompressionType: null
- Files:
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
- size: 176
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2
- size: 168
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3
- size: 192
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4
- size: 168
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5
- size: 192
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6
- size: 216
-AO_Schema:
-- name: i
- type: int4
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
- dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
- (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED BY (i)
-Encoding: UTF8
-FileFormat: AO
-TableName: public.t_1
-Version: 1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
deleted file mode 100644
index 1a2a459..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_2.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
- dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
- (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED BY (i)
-Encoding: UTF8
-FileFormat: Parquet
-Parquet_FileLocations:
- Checksum: false
- CompressionLevel: 0
- CompressionType: null
- EnableDictionary: false
- Files:
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
- size: 254
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/2
- size: 250
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/3
- size: 258
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/4
- size: 250
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/5
- size: 258
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/6
- size: 270
- PageSize: 1048576
- RowGroupSize: 8388608
-Parquet_Schema:
-- name: i
- type: int4
-TableName: public.t_2
-Version: 1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
deleted file mode 100644
index df6730e..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_3.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-AO_FileLocations:
- Blocksize: 32768
- Checksum: false
- CompressionLevel: 0
- CompressionType: null
- Files:
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
- size: 1016
-AO_Schema:
-- name: i
- type: int4
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
- dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
- (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED RANDOMLY
-Encoding: UTF8
-FileFormat: AO
-TableName: public.t_3
-Version: 1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml b/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
deleted file mode 100644
index 8df2afc..0000000
--- a/src/test/feature/ManagementTool/usage2case3/t_tpl_old_4.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-Bucketnum: 6
-DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
- dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
- (based on LLVM 3.6.0svn) compiled on Jul 6 2016 10:22:33
-DFS_URL: hdfs://@PORT@
-Distribution_Policy: DISTRIBUTED RANDOMLY
-Encoding: UTF8
-FileFormat: Parquet
-Parquet_FileLocations:
- Checksum: false
- CompressionLevel: 0
- CompressionType: null
- EnableDictionary: false
- Files:
- - path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID@/1
- size: 596
- PageSize: 1048576
- RowGroupSize: 8388608
-Parquet_Schema:
-- name: i
- type: int4
-TableName: public.t_4
-Version: 1.0.0
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/778ab999/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 67a188d..b9b759e 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -18,7 +18,7 @@
# under the License.
# Usage1: hawq register [-h hostname] [-p port] [-U username] [-d database] [-f filepath] [-e eof] <tablename>
-# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force] [--repair] <tablename>
+# Usage2: hawq register [-h hostname] [-p port] [-U username] [-d database] [-c config] [--force] <tablename>
import os
import sys
@@ -54,7 +54,6 @@ def option_parser():
parser.add_option('-e', '--eof', dest='filesize', type='int', default=None, help='eof of the file to be registered')
parser.add_option('-c', '--config', dest='yml_config', default='', help='configuration file in YAML format')
parser.add_option('-F', '--force', dest='force', action='store_true', default=False)
- parser.add_option('-R', '--repair', dest='repair', action='store_true', default=False)
return parser
@@ -340,7 +339,7 @@ class HawqRegister(object):
self.accessor = GpRegisterAccessor(conn)
self.utility_accessor = GpRegisterAccessor(utility_conn)
self.failure_handler = failure_handler
- self.mode = self._init_mode(options.force, options.repair)
+ self.mode = self._init_mode(options.force)
self.srcfiles = []
self.dstfiles = []
self.files_same_path = []
@@ -351,20 +350,13 @@ class HawqRegister(object):
self.eofuncompresseds_same_path = []
self.segnos_same_path = []
- def _init_mode(self, force, repair):
+ def _init_mode(self, force):
def table_existed():
return self.accessor.get_table_existed(self.dst_table_name)
if self.yml:
if force:
return 'force'
- elif repair:
- logger.info('--repair mode is not supported in current release of hawq register.')
- sys.exit(0)
- if not table_existed():
- logger.error('--repair mode asserts the table has been already created.')
- sys.exit(1)
- return 'repair'
else:
return 'usage2_table_not_exist'
else:
@@ -583,9 +575,8 @@ class HawqRegister(object):
self.filepath = self.files[0][:self.files[0].rfind('/')] if self.files else ''
self._check_file_not_folder()
self._check_database_encoding()
- if self.mode != 'repair':
- if not self._create_table() and self.mode != 'force':
- self.mode = 'usage2_table_exist'
+ if not self._create_table() and self.mode != 'force':
+ self.mode = 'usage2_table_exist'
else:
if self._is_folder(self.filepath) and self.filesize:
logger.error('-e option is only supported with single file case.')
@@ -628,23 +619,6 @@ class HawqRegister(object):
logger.error("Hawq aoseg metadata doesn't consistent with file numbers in hdfs.")
self.failure_handler.rollback()
sys.exit(1)
- if self.mode == 'repair':
- if self.tabledir.strip('/') != self.filepath.strip('/'):
- logger.error("In repair mode, file path from yaml file should be the same with table's path.")
- self.failure_handler.rollback()
- sys.exit(1)
- existed_info = {}
- for k, fn in enumerate(existed_files):
- existed_info[fn] = existed_sizes[k]
- for k, fn in enumerate(self.files):
- if fn not in existed_files:
- logger.error('Can not register in repair mode since giving non-existing file: %s.' % fn)
- self.failure_handler.rollback()
- sys.exit(1)
- if self.sizes[k] > existed_info[fn]:
- logger.error('Can not register in repair mode since giving larger file size: %s' % self.sizes[k])
- self.failure_handler.rollback()
- sys.exit(1)
if self.mode == 'usage2_table_exist':
if self.tabledir.strip('/') == self.filepath.strip('/'):
@@ -689,14 +663,6 @@ class HawqRegister(object):
logger.error('In force mode, you should include existing table files in yaml configuration file. Otherwise you should drop the previous table before register --force.')
self.failure_handler.rollback()
sys.exit(1)
- elif self.mode == 'repair':
- self.do_not_move = True
- self.files_update, self.sizes_update = [fn for fn in self.files], [sz for sz in self.sizes]
- self.files_delete = []
- for fn in existed_files:
- if fn not in self.files:
- self.files_delete.append(fn)
- self.files_append, self.sizes_append = [], []
self._check_files_and_table_in_same_hdfs_cluster(self.filepath, self.tabledir)
@@ -963,7 +929,7 @@ class HawqRegister(object):
def _prepare_register(self):
if not self.do_not_move:
self._set_move_files_in_hdfs()
- if self.mode == 'force' or self.mode == 'repair':
+ if self.mode == 'force':
self._set_modify_metadata('force')
else:
self._set_modify_metadata('insert')
@@ -972,10 +938,6 @@ class HawqRegister(object):
if not self.do_not_move:
self._move_files_in_hdfs()
self._modify_metadata()
- if self.mode == 'repair':
- if self.files_delete:
- self._delete_files_in_hdfs()
- self._delete_metadata()
logger.info('Hawq Register Succeed.')
@@ -1033,11 +995,10 @@ class HawqRegisterPartition(HawqRegister):
self._check_file_not_folder()
if self.yml:
self._check_database_encoding()
- if self.mode != 'repair':
- if not self._create_table() and self.mode != 'force':
- self.mode = 'usage2_table_exist'
- self._check_partitionby()
- self._check_partition_num()
+ if not self._create_table() and self.mode != 'force':
+ self.mode = 'usage2_table_exist'
+ self._check_partitionby()
+ self._check_partition_num()
partitions = self._get_partition_info()
self.queries = "set allow_system_table_mods='dml';"
self.queries += "begin transaction;"
@@ -1124,10 +1085,10 @@ def test(options, args):
if __name__ == '__main__':
parser = option_parser()
options, args = parser.parse_args()
- if len(args) != 1 or (options.force and options.repair):
+ if len(args) != 1:
parser.print_help(sys.stderr)
sys.exit(1)
- if (options.yml_config or options.force or options.repair) and options.filepath:
+ if (options.yml_config or options.force) and options.filepath:
parser.print_help(sys.stderr)
sys.exit(1)
if local_ssh('hdfs'):