You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2016/10/18 05:38:27 UTC
incubator-hawq git commit: HAWQ-1099. Output yaml file should not
contain Bucketnum attribute with random distributed table.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 676eb828c -> c8ef3a0ac
HAWQ-1099. Output yaml file should not contain Bucketnum attribute with random distributed table.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c8ef3a0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c8ef3a0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c8ef3a0a
Branch: refs/heads/master
Commit: c8ef3a0ac847743dbed0d0715b6534e34f95d92a
Parents: 676eb82
Author: xunzhang <xu...@gmail.com>
Authored: Thu Oct 13 18:25:40 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Oct 18 13:37:51 2016 +0800
----------------------------------------------------------------------
tools/bin/hawqextract | 6 ++++--
tools/bin/hawqregister | 15 ++++++++++-----
2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqextract
----------------------------------------------------------------------
diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract
index 28f55d6..3a6ef66 100644
--- a/tools/bin/hawqextract
+++ b/tools/bin/hawqextract
@@ -421,7 +421,8 @@ def extract_metadata(conn, tbname):
logger.info('-- extract Distribution_Policy')
metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'], relid)
logger.info('-- extract bucket number')
- metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
+ if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED BY'):
+ metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
def extract_Parquet_metadata():
relid = rel_pgclass['oid']
@@ -471,7 +472,8 @@ def extract_metadata(conn, tbname):
logger.info('-- extract Distribution_Policy')
metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'], relid)
logger.info('-- extract bucket number')
- metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
+ if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED BY'):
+ metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
# extract AO/Parquet specific metadata
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 29b3a30..a1c8e29 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -60,14 +60,18 @@ def option_parser():
def register_yaml_dict_check(D, table_column_num, src_tablename):
'''check exists'''
- check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName', 'Bucketnum']
+ check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName']
for attr in check_list:
if D.get(attr) is None:
logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n See example in "hawq register --help".' % attr)
sys.exit(1)
- if D['Bucketnum'] <= 0:
- logger.error('Bucketnum should not be zero, please check your yaml configuration file.')
- sys.exit(1)
+ if D['Distribution_Policy'].startswith('DISTRIBUTED BY'):
+ if D.get('Bucketnum') is None:
+ logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n See example in "hawq register --help".' % attr)
+ sys.exit(1)
+ if D['Bucketnum'] <= 0:
+ logger.error('Bucketnum should not be zero, please check your yaml configuration file.')
+ sys.exit(1)
if D['FileFormat'] in ['Parquet', 'AO']:
prefix = D['FileFormat']
local_check_list = ['%s_FileLocations' % prefix, '%s_Schema' % prefix]
@@ -432,7 +436,8 @@ class HawqRegister(object):
if len(params[Format_FileLocations]['Files']):
files, sizes = [params['DFS_URL'] + d['path'] for d in params[Format_FileLocations]['Files']], [d['size'] for d in params[Format_FileLocations]['Files']]
encoding = params['Encoding']
- self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], params['Bucketnum'], partitionby,\
+ bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED BY') else 6
+ self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations], bucketNum, partitionby,\
partitions_constraint, partitions_name, partitions_compression_level, partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes, encoding)