You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by sh...@apache.org on 2016/03/21 23:01:30 UTC
[01/50] incubator-hawq git commit: HAWQ-527. Remove dead code of
generateResourceRefreshHeartBeat()
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-459 b1198ed9c -> 55be00cee
HAWQ-527. Remove dead code of generateResourceRefreshHeartBeat()
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b3e2add6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b3e2add6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b3e2add6
Branch: refs/heads/HAWQ-459
Commit: b3e2add6d346ecde3c03307f27fcdcf006c1fa70
Parents: fa993ac
Author: YI JIN <yj...@pivotal.io>
Authored: Mon Mar 14 16:48:55 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Mon Mar 14 16:48:55 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/communication/rmcomm_QD2RM.c | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b3e2add6/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index a645dd4..d601383 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -1519,12 +1519,6 @@ void *generateResourceRefreshHeartBeat(void *arg)
pg_usleep(rm_session_lease_heartbeat_interval * 1000000L);
}
- if ( fd >= 0 )
- {
- close(fd);
- fd = -1;
- }
-
freeHeartBeatThreadArg(&tharg);
write_log("generateResourceRefreshHeartBeat exits.");
return 0;
[19/50] incubator-hawq git commit: HAWQ-531. Change GUC default value.
Posted by sh...@apache.org.
HAWQ-531. Change GUC default value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d1a8540
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d1a8540
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d1a8540
Branch: refs/heads/HAWQ-459
Commit: 4d1a85405b87145fb931a16be9e22bd40f16eca8
Parents: 96345af
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 11:08:57 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 11:08:57 2016 +0800
----------------------------------------------------------------------
src/backend/utils/misc/guc.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d1a8540/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fa58f86..ba8e747 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6379,7 +6379,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_limit,
- 1000, 1, 65535, NULL, NULL
+ 512, 1, 65535, NULL, NULL
},
{
@@ -6535,7 +6535,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_rejectrequest_nseg_limit,
- 2, 0, 65535, NULL, NULL
+ 4, 0, 65535, NULL, NULL
},
{
[25/50] incubator-hawq git commit: Revert "HAWQ-547. When HAWQ RM
does not have valid YARN resource queue status received the cluster report is
not accepted"
Posted by sh...@apache.org.
Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted"
This reverts commit 5fda1826aff4780b20e24db5210f79f867ef39ae.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2be5899c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2be5899c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2be5899c
Branch: refs/heads/HAWQ-459
Commit: 2be5899ca55b954de90aa72d00c8bd02f8f2fb83
Parents: aef7bd2
Author: YI JIN <yj...@pivotal.io>
Authored: Thu Mar 17 16:41:40 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Thu Mar 17 16:41:40 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/include/errorcode.h | 1 -
.../resourcebroker/resourcebroker_LIBYARN.c | 18 ++----------------
2 files changed, 2 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2be5899c/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index de08fff..ca338a8 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,7 +162,6 @@ enum DRM_ERROR_CODE {
RESBROK_WRONG_GLOB_MGR_ADDRESS,
RESBROK_WRONG_GLOB_MGR_QUEUE,
RESBROK_WRONG_GLOB_MGR_APPNAME,
- RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
RESBROK_PIPE_ERROR,
RESBROK_NOMORE_RESOURCE_IN_GRM,
RESBROK_WRONG_MESSAGE_ID,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2be5899c/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index b8d205e..cdbe0d5 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
}
elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
- "maximum queue capacity is %lf.",
+ "maximum capacity is %lf.",
response.MachineCount,
response.QueueMaxCapacity);
@@ -631,19 +631,6 @@ int handleRB2RM_ClusterReport(void)
* TILL NOW, the whole message content is received.
*/
- if ( res == FUNC_RETURN_OK )
- {
- /* Check if the YARN resource queue report is valid, i.e. maximum
- * capacity and capacity are all greater than 0.
- */
- if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
- {
- elog(WARNING, "YARN mode resource broker got invalid cluster report");
- res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
-
- }
- }
-
/* If something wrong, no need to keep the received content, free them. */
if ( res != FUNC_RETURN_OK )
{
@@ -1189,7 +1176,6 @@ void RB_LIBYARN_handleError(int errorcode)
}
else
{
- Assert(errorcode == FUNC_RETURN_OK ||
- errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
+ Assert(errorcode == FUNC_RETURN_OK);
}
}
[35/50] incubator-hawq git commit: HAWQ-531. Change GUC default value.
Posted by sh...@apache.org.
HAWQ-531. Change GUC default value.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/eaaf9451
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/eaaf9451
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/eaaf9451
Branch: refs/heads/HAWQ-459
Commit: eaaf94514396c8f60716a78a76852017f7614808
Parents: 4d997b5
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:16:51 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800
----------------------------------------------------------------------
src/backend/utils/misc/guc.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eaaf9451/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 3d13eb4..a6a8bee 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4516,7 +4516,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&default_hash_table_bucket_number,
- 6, 1, INT_MAX, NULL, NULL
+ 6, 1, 10000, NULL, NULL
},
{
@@ -4525,7 +4525,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_copy_from_perquery,
- 6, 1, INT_MAX, NULL, NULL
+ 6, 1, 10000, NULL, NULL
},
{
@@ -4534,7 +4534,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_analyze_perquery_perseg_limit,
- 4, 1, INT_MAX, NULL, NULL
+ 4, 1, 10000, NULL, NULL
},
{
@@ -4543,7 +4543,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_analyze_perquery_limit,
- 256, 1, INT_MAX, NULL, NULL
+ 256, 1, 10000, NULL, NULL
},
{
@@ -6388,7 +6388,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_limit,
- 1000, 1, 65535, NULL, NULL
+ 512, 1, 10000, NULL, NULL
},
{
@@ -6398,7 +6398,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_perseg_limit,
- 6, 1, 65535, NULL, NULL
+ 6, 1, 10000, NULL, NULL
},
{
[28/50] incubator-hawq git commit: HAWQ-551. Add datalocility time to
explain analyse result
Posted by sh...@apache.org.
HAWQ-551. Add datalocility time to explain analyse result
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/dfc67313
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/dfc67313
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/dfc67313
Branch: refs/heads/HAWQ-459
Commit: dfc673135f9f1c92487423436428be82ad1db1e4
Parents: 94b7a82
Author: ivan <iw...@pivotal.io>
Authored: Thu Mar 17 15:07:27 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Fri Mar 18 09:32:31 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdatalocality.c | 28 +++++++++++++++++++++++++---
src/backend/commands/explain.c | 2 +-
src/backend/optimizer/plan/planner.c | 1 +
src/include/cdb/cdbdatalocality.h | 1 +
src/include/nodes/plannodes.h | 1 +
5 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index c1f829b..eec87b4 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -309,6 +309,10 @@ typedef struct split_to_segment_mapping_context {
int64 total_file_count;
int64 total_metadata_logic_len;
+
+ int metadata_cache_time_us;
+ int alloc_resource_time_us;
+ int cal_datalocality_time_us;
} split_to_segment_mapping_context;
typedef struct vseg_list{
@@ -569,6 +573,10 @@ static void init_datalocality_context(split_to_segment_mapping_context *context)
context->total_split_count = 0;
context->total_file_count = 0;
context->total_metadata_logic_len = 0;
+
+ context->metadata_cache_time_us = 0;
+ context->alloc_resource_time_us = 0;
+ context->cal_datalocality_time_us = 0;
return;
}
@@ -928,6 +936,8 @@ int64 get_block_locations_and_claculte_table_size(split_to_segment_mapping_conte
}
context->total_file_count = totalFileCount;
context->total_size = total_size;
+
+ context->metadata_cache_time_us = eclaspeTime;
if(debug_datalocality_time){
elog(LOG, "metadata overall execution time: %d us. \n", eclaspeTime);
@@ -3395,13 +3405,13 @@ static void print_datalocality_overall_log_information(SplitAllocResult *result,
appendStringInfo(result->datalocalityInfo, "data locality ratio: %.3f; virtual segment number: %d; "
"different host number: %d; virtual segment number per host(avg/min/max): (%d/%d/%d); "
"segment size(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); "
- "segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f)."
+ "segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f); "
,log_context->datalocalityRatio,assignment_context->virtual_segment_num,log_context->numofDifferentHost,
log_context->avgSegmentNumofHost,log_context->minSegmentNumofHost,log_context->maxSegmentNumofHost,
log_context->avgSizeOverall,log_context->minSizeSegmentOverall,log_context->maxSizeSegmentOverall,
log_context->avgSizeOverallPenalty,log_context->minSizeSegmentOverallPenalty,log_context->maxSizeSegmentOverallPenalty,
- log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall
- );
+ log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall);
+
if (debug_fake_datalocality) {
fprintf(fp, "datalocality ratio: %.3f; virtual segments number: %d, "
"different host number: %d, segment number per host(avg/min/max): (%d/%d/%d); "
@@ -3919,9 +3929,17 @@ run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments, Query
uint64_t run_datalocality = 0;
run_datalocality = gettime_microsec();
int dl_overall_time = run_datalocality - before_run_allocation;
+
+ context->cal_datalocality_time_us = dl_overall_time;
+
if(debug_datalocality_time){
elog(LOG, "datalocality overall execution time: %d us. \n", dl_overall_time);
}
+
+ result->datalocalityTime = (double)(context->metadata_cache_time_us + context->alloc_resource_time_us + context->cal_datalocality_time_us)/ 1000;
+ appendStringInfo(result->datalocalityInfo, "DFS metadatacache: %.3f ms; resource allocation: %.3f ms; datalocality calculation: %.3f ms.",
+ (double)context->metadata_cache_time_us/1000, (double)context->alloc_resource_time_us/1000, (double)context->cal_datalocality_time_us/1000);
+
return alloc_result;
}
@@ -3980,6 +3998,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
result->relsType = NIL;
result->planner_segments = -1;
result->datalocalityInfo = makeStringInfo();
+ result->datalocalityTime = 0;
/* fake data locality */
if (debug_fake_datalocality) {
@@ -4230,6 +4249,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
}
uint64_t after_rm_allocate_resource = gettime_microsec();
int eclaspeTime = after_rm_allocate_resource - before_rm_allocate_resource;
+
+ context.alloc_resource_time_us = eclaspeTime;
+
if(debug_datalocality_time){
elog(LOG, "rm allocate resource overall execution time: %d us. \n", eclaspeTime);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/commands/explain.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 51afde2..0e7bff5 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -470,7 +470,7 @@ ExplainOnePlan_internal(PlannedStmt *plannedstmt,
MemoryContext oldcxt = CurrentMemoryContext;
QueryDesc *queryDesc;
instr_time starttime;
- double totaltime = 0;
+ double totaltime = plannedstmt->datalocalityTime / 1000;
StringInfo buf = &es->outbuf;
EState *estate = NULL;
int eflags;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index be057e0..6b2e13e 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -459,6 +459,7 @@ planner(Query *parse, int cursorOptions,
result->scantable_splits = ppResult->saResult.alloc_results;
result->planner_segments = ppResult->saResult.planner_segments;
result->datalocalityInfo = ppResult->saResult.datalocalityInfo;
+ result->datalocalityTime = ppResult->saResult.datalocalityTime;
}
plannerLevel--;
if ((ppResult != NULL))
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 95f2bd5..d28fc3e 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -46,6 +46,7 @@ typedef struct SplitAllocResult
int planner_segments;
List *relsType;// relation type after datalocality changing
StringInfo datalocalityInfo;
+ double datalocalityTime;
} SplitAllocResult;
/*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/nodes/plannodes.h
----------------------------------------------------------------------
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index ebccb9d..7c10cd5 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -204,6 +204,7 @@ typedef struct PlannedStmt
MemoryAccount *memoryAccount;
StringInfo datalocalityInfo;
+ double datalocalityTime;
} PlannedStmt;
[49/50] incubator-hawq git commit: HAWQ-566. Fix analyze and truncate
hung.
Posted by sh...@apache.org.
HAWQ-566. Fix analyze and truncate hung.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/76dc6319
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/76dc6319
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/76dc6319
Branch: refs/heads/HAWQ-459
Commit: 76dc63191fb21b1321c00f00954afc1cb98c6f45
Parents: 7daee40
Author: hzhang2 <zh...@163.com>
Authored: Mon Mar 21 16:38:15 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:38:15 2016 +0800
----------------------------------------------------------------------
src/backend/commands/analyze.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/76dc6319/src/backend/commands/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 32b3416..ac12837 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -828,7 +828,7 @@ static int calculate_virtual_segment_number(List* candidateOids) {
candidateOid = lfirst_oid(le1);
//Relation rel = (Relation)lfirst(le1);
- Relation rel = relation_open(candidateOid, AccessShareLock);
+ Relation rel = relation_open(candidateOid, ShareUpdateExclusiveLock);
if (candidateOid > 0 ) {
GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
candidateOid);
@@ -848,7 +848,7 @@ static int calculate_virtual_segment_number(List* candidateOids) {
totalDataSize += calculate_relation_size(rel);
}
}
- relation_close(rel, AccessShareLock);
+ relation_close(rel, NoLock);
}
if (isHashRelationExist) {
[40/50] incubator-hawq git commit: HAWQ-532. Optimise vseg number for
copy to statement.
Posted by sh...@apache.org.
HAWQ-532. Optimise vseg number for copy to statement.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/36571bce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/36571bce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/36571bce
Branch: refs/heads/HAWQ-459
Commit: 36571bce2c96fce04ceb94f9ab78ace4cc90c66d
Parents: b3a3521
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 12:36:39 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 12:36:39 2016 +0800
----------------------------------------------------------------------
src/backend/commands/copy.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/36571bce/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 2f4cf39..e37fb83 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1888,11 +1888,12 @@ DoCopyTo(CopyState cstate)
*/
PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
- Assert(pn);
List *lFullRelOids = NIL;
- lFullRelOids = all_leaf_partition_relids(pn);
- lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id); /* root partition */
- lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+ if(pn){
+ lFullRelOids = all_leaf_partition_relids(pn);
+ lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+ }
+ lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
target_segment_num = calculate_virtual_segment_number(lFullRelOids);
elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
[48/50] incubator-hawq git commit: HAWQ-562. Refactor bucket number
of external table.
Posted by sh...@apache.org.
HAWQ-562. Refactor bucket number of external table.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7daee400
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7daee400
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7daee400
Branch: refs/heads/HAWQ-459
Commit: 7daee4002780797587c3dcca265c646fd0c92e1e
Parents: 695606f
Author: hzhang2 <zh...@163.com>
Authored: Mon Mar 21 10:26:11 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:29:42 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbcopy.c | 2 +-
src/backend/cdb/cdbdatalocality.c | 97 ++++++++++++++++++++--------------
src/backend/cdb/cdbfilesplit.c | 67 +++++++----------------
src/backend/commands/tablecmds.c | 11 ++--
src/backend/parser/analyze.c | 2 +-
src/include/access/filesplit.h | 2 +-
src/include/cdb/cdbdatalocality.h | 1 -
7 files changed, 88 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbcopy.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbcopy.c b/src/backend/cdb/cdbcopy.c
index 17ffa96..ee2ff80 100644
--- a/src/backend/cdb/cdbcopy.c
+++ b/src/backend/cdb/cdbcopy.c
@@ -222,7 +222,7 @@ cdbCopyStart(CdbCopy *c, char *copyCmd, Oid relid, Oid relerror, List *err_aoseg
List *scantable_splits = NIL;
prepareDispatchedCatalogRelation(q->contextdisp, relid, FALSE, NULL);
scantable_splits = AssignAOSegFileSplitToSegment(relid, NIL,
- true, c->partition_num,
+ c->partition_num,
scantable_splits);
((CopyStmt *)q->utilityStmt)->scantable_splits = scantable_splits;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index 3f8fcb5..4c67429 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -34,6 +34,7 @@
#include "access/parquetsegfiles.h"
#include "catalog/catalog.h"
#include "catalog/catquery.h"
+#include "catalog/pg_exttable.h"
#include "catalog/pg_inherits.h"
#include "catalog/pg_proc.h"
#include "cdb/cdbdatalocality.h"
@@ -296,11 +297,13 @@ typedef struct split_to_segment_mapping_context {
int64 split_size;
MemoryContext old_memorycontext;
MemoryContext datalocality_memorycontext;
- int externTableSegNum; //expected virtual segment number when external table exists
+ int externTableOnClauseSegNum; //expected virtual segment number when external table exists
+ int externTableLocationSegNum; //expected virtual segment number when external table exists
int tableFuncSegNum; //expected virtual segment number when table function exists
int hashSegNum; // expected virtual segment number when there is hash table in from clause
int randomSegNum; // expected virtual segment number when there is random table in from clause
int resultRelationHashSegNum; // expected virtual segment number when hash table as a result relation
+ int minimum_segment_num; //default is 1.
int64 randomRelSize; //all the random relation size
int64 hashRelSize; //all the hash relation size
@@ -529,13 +532,15 @@ static void init_datalocality_context(split_to_segment_mapping_context *context)
context->rtc_context.range_tables = NIL;
context->rtc_context.full_range_tables = NIL;
- context->externTableSegNum = 0;
+ context->externTableOnClauseSegNum = 0;
+ context->externTableLocationSegNum = 0;
context->tableFuncSegNum = 0;
context->hashSegNum = 0;
context->resultRelationHashSegNum = 0;
context->randomSegNum = 0;
context->randomRelSize = 0;
context->hashRelSize = 0;
+ context->minimum_segment_num = 1;
/*
* initialize the data distribution
* static context.
@@ -762,17 +767,24 @@ static void check_keep_hash_and_external_table(
/* targetPolicy->bucketnum is bucket number of external table,
* whose default value is set to default_segment_num
*/
- if (context->externTableSegNum == 0) {
- context->externTableSegNum = targetPolicy->bucketnum;
- } else {
- if (context->externTableSegNum < targetPolicy->bucketnum) {
- context->externTableSegNum = targetPolicy->bucketnum;
- /*
- * In this case, two external table join but with different bucket number
- * we cannot allocate the right segment number.
- */
- //now we just restrict that vseg num > bucket number of external table
- //elog(ERROR, "All external tables in one query must have the same bucket number!");
+ ExtTableEntry* extEnrty = GetExtTableEntry(rel->rd_id);
+ if(extEnrty->isweb){
+ if (context->externTableOnClauseSegNum == 0) {
+ context->externTableOnClauseSegNum = targetPolicy->bucketnum;
+ } else {
+ if (context->externTableOnClauseSegNum != targetPolicy->bucketnum) {
+ /*
+ * In this case, two external table join but with different bucket number
+ * we cannot allocate the right segment number.
+ */
+ elog(ERROR, "All external tables in one query must have the same bucket number!");
+ }
+ }
+ }
+ else{
+ if (context->externTableLocationSegNum < targetPolicy->bucketnum) {
+ context->externTableLocationSegNum = targetPolicy->bucketnum;
+ context->minimum_segment_num = targetPolicy->bucketnum;
}
}
}
@@ -4074,17 +4086,17 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
&(context.rtc_context.range_tables), &isTableFunctionExists,
context.datalocality_memorycontext);
- /*Table Function VSeg Number = default_segment_number(configured in GUC) if table function exists,
+ /* set expected virtual segment number for hash table and external table*/
+ /* calculate hashSegNum, externTableSegNum, resultRelationHashSegNum */
+ check_keep_hash_and_external_table(&context, query, intoPolicy);
+
+ /*Table Function VSeg Number = default_segment_number(configured in GUC) if table function exists or gpfdist exists,
*0 Otherwise.
*/
if (isTableFunctionExists) {
context.tableFuncSegNum = GetUserDefinedFunctionVsegNum();
}
- /* set expected virtual segment number for hash table and external table*/
- /* calculate hashSegNum, externTableSegNum, resultRelationHashSegNum */
- check_keep_hash_and_external_table(&context, query, intoPolicy);
-
/* get block location and calculate relation size*/
get_block_locations_and_claculte_table_size(&context);
@@ -4163,13 +4175,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
context.randomSegNum = expected_segment_num_with_max_filecount;
}
/* Step4 we at least use one segment*/
- if (context.randomSegNum < minimum_segment_num) {
- context.randomSegNum = minimum_segment_num;
+ if (context.randomSegNum < context.minimum_segment_num) {
+ context.randomSegNum = context.minimum_segment_num;
}
int maxExpectedNonRandomSegNum = 0;
- if (maxExpectedNonRandomSegNum < context.externTableSegNum)
- maxExpectedNonRandomSegNum = context.externTableSegNum;
if (maxExpectedNonRandomSegNum < context.tableFuncSegNum)
maxExpectedNonRandomSegNum = context.tableFuncSegNum;
if (maxExpectedNonRandomSegNum < context.hashSegNum)
@@ -4183,7 +4193,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
fprintf(fpsegnum, "Result relation hash segment num : %d.\n", context.resultRelationHashSegNum);
fprintf(fpsegnum, "\n");
fprintf(fpsegnum, "Table function segment num : %d.\n", context.tableFuncSegNum);
- fprintf(fpsegnum, "Extern table segment num : %d.\n", context.externTableSegNum);
+ fprintf(fpsegnum, "Extern table segment num : %d.\n", context.externTableOnClauseSegNum);
fprintf(fpsegnum, "From hash relation segment num : %d.\n", context.hashSegNum);
fprintf(fpsegnum, "MaxExpectedNonRandom segment num : %d.\n", maxExpectedNonRandomSegNum);
fprintf(fpsegnum, "\n");
@@ -4193,29 +4203,33 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
int maxTargetSegmentNumber = 0;
/* we keep resultRelationHashSegNum in the highest priority*/
if (context.resultRelationHashSegNum != 0) {
- if (context.resultRelationHashSegNum < context.externTableSegNum
- && context.externTableSegNum != 0) {
+ if ((context.resultRelationHashSegNum != context.externTableOnClauseSegNum
+ && context.externTableOnClauseSegNum != 0)
+ || (context.resultRelationHashSegNum < context.externTableLocationSegNum)) {
cleanup_allocation_algorithm(&context);
elog(ERROR, "Could not allocate enough memory! "
"bucket number of result hash table and external table should match each other");
}
maxTargetSegmentNumber = context.resultRelationHashSegNum;
minTargetSegmentNumber = context.resultRelationHashSegNum;
- } else if (maxExpectedNonRandomSegNum > 0) {
+ }
+ else if(context.externTableOnClauseSegNum > 0){
/* bucket number of external table must be the same with the number of virtual segments*/
- if (maxExpectedNonRandomSegNum == context.externTableSegNum) {
- context.externTableSegNum =
- context.externTableSegNum < minimum_segment_num ?
- minimum_segment_num : context.externTableSegNum;
- maxTargetSegmentNumber = context.externTableSegNum;
- minTargetSegmentNumber = context.externTableSegNum;
- } else if (maxExpectedNonRandomSegNum == context.hashSegNum) {
+ if(context.externTableOnClauseSegNum < context.externTableLocationSegNum){
+ cleanup_allocation_algorithm(&context);
+ elog(ERROR, "external table bucket number should match each other");
+ }
+ maxTargetSegmentNumber = context.externTableOnClauseSegNum;
+ minTargetSegmentNumber = context.externTableOnClauseSegNum;
+ }
+ else if (maxExpectedNonRandomSegNum > 0) {
+ if (maxExpectedNonRandomSegNum == context.hashSegNum) {
/* in general, we keep bucket number of hash table equals to the number of virtual segments
* but this rule can be broken when there is a large random table in the range tables list
*/
context.hashSegNum =
- context.hashSegNum < minimum_segment_num ?
- minimum_segment_num : context.hashSegNum;
+ context.hashSegNum < context.minimum_segment_num ?
+ context.minimum_segment_num : context.hashSegNum;
double considerRandomWhenHashExistRatio = 1.5;
/*if size of random table >1.5 *hash table, we consider relax the restriction of hash bucket number*/
if (context.randomRelSize
@@ -4224,7 +4238,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
context.randomSegNum = context.hashSegNum;
}
maxTargetSegmentNumber = context.randomSegNum;
- minTargetSegmentNumber = minimum_segment_num;
+ minTargetSegmentNumber = context.minimum_segment_num;
} else {
maxTargetSegmentNumber = context.hashSegNum;
minTargetSegmentNumber = context.hashSegNum;
@@ -4232,17 +4246,17 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
} else if (maxExpectedNonRandomSegNum == context.tableFuncSegNum) {
/* if there is a table function, we should at least use tableFuncSegNum virtual segments*/
context.tableFuncSegNum =
- context.tableFuncSegNum < minimum_segment_num ?
- minimum_segment_num : context.tableFuncSegNum;
+ context.tableFuncSegNum < context.minimum_segment_num ?
+ context.minimum_segment_num : context.tableFuncSegNum;
if (context.randomSegNum < context.tableFuncSegNum) {
context.randomSegNum = context.tableFuncSegNum;
}
maxTargetSegmentNumber = context.randomSegNum;
- minTargetSegmentNumber = minimum_segment_num;
+ minTargetSegmentNumber = context.minimum_segment_num;
}
} else {
maxTargetSegmentNumber = context.randomSegNum;
- minTargetSegmentNumber = minimum_segment_num;
+ minTargetSegmentNumber = context.minimum_segment_num;
}
if (enforce_virtual_segment_number > 0) {
@@ -4254,6 +4268,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
maxTargetSegmentNumber = fixedVsegNum;
minTargetSegmentNumber = fixedVsegNum;
}
+ if(maxTargetSegmentNumber < minTargetSegmentNumber){
+ maxTargetSegmentNumber = minTargetSegmentNumber;
+ }
uint64_t before_rm_allocate_resource = gettime_microsec();
/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbfilesplit.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbfilesplit.c b/src/backend/cdb/cdbfilesplit.c
index 4a2ad04..128f949 100644
--- a/src/backend/cdb/cdbfilesplit.c
+++ b/src/backend/cdb/cdbfilesplit.c
@@ -44,8 +44,7 @@ static void fileSplit_free(FileSplitNode *split);
static List *
computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits,
- List *segment_infos, bool keep_hash_policy,
- int target_segment_num);
+ List *segment_infos, int target_segment_num);
static List *
postProcessSplitsPerSegment(List *oldSplitToSegmentMaps);
@@ -54,7 +53,7 @@ static int
fileSplitNodeCmp(const void *left, const void *right);
static SegFileSplitMapNode *
-AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num)
+AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num)
{
SegFileSplitMapNode *result = NULL;
char storageChar;
@@ -108,7 +107,7 @@ AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_ha
* data locality into account.
*/
splitToSegmentMaps = computeSplitToSegmentMaps(relid, targetPolicy, splits, segment_infos,
- keep_hash_policy, target_segment_num);
+ target_segment_num);
Assert(splitToSegmentMaps);
list_free(splits);
@@ -118,7 +117,7 @@ AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_ha
}
List *
-AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num, List *existings)
+AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num, List *existings)
{
SegFileSplitMapNode *result = NULL;
@@ -130,7 +129,7 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
foreach (child, children)
{
Oid myrelid = lfirst_oid(child);
- result = AssignSingleAOSegFileSplitToSegment(myrelid, segment_infos, keep_hash_policy,
+ result = AssignSingleAOSegFileSplitToSegment(myrelid, segment_infos,
target_segment_num);
existings = lappend(existings, result);
}
@@ -140,7 +139,7 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
}
else
{
- result = AssignSingleAOSegFileSplitToSegment(relid, segment_infos, keep_hash_policy,
+ result = AssignSingleAOSegFileSplitToSegment(relid, segment_infos,
target_segment_num);
existings = lappend(existings, result);
}
@@ -149,25 +148,14 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
}
/*
- * If keep_hash_policy is false, then the table is treated as randomly
- * distributed, regardless of its distribution policy.
*
* If segment_infos is NIL, then data locality is not needed.
*/
static List *
computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits, List *segment_infos,
- bool keep_hash_policy, int target_segment_num)
+ int target_segment_num)
{
List *splitToSegmentMaps = NIL;
- if (keep_hash_policy)
- {
- /*
- * If we want to keep the hash distribution policy,
- * we need to make sure the data partition number
- * equals to the number of target segments.
- */
- Assert(targetPolicy->bucketnum == target_segment_num);
- }
if (segment_infos != NIL)
{
@@ -184,37 +172,22 @@ computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits, List
*/
if (segment_infos == NIL)
{
- /*
- * Sub case 1: we need to keep the hash distribution policy.
- *
- * In this case, we just need consider the segment file number.
- * This case works exactly the same as HAWQ1.x.
- */
- if (keep_hash_policy)
+ ListCell *lc;
+ int i;
+ for (i = 0; i < target_segment_num; i++)
{
- ListCell *lc;
- int i;
- for (i = 0; i < target_segment_num; i++)
- {
- splitToSegmentMaps = lappend(splitToSegmentMaps, NIL);
- }
-
- foreach(lc, splits)
- {
- int assigned_seg_no;
- ListCell *per_seg_split;
- FileSplit split = (FileSplitNode *)lfirst(lc);
- Assert(split);
- assigned_seg_no = (split->segno - 1) % target_segment_num;
- per_seg_split = list_nth_cell(splitToSegmentMaps, assigned_seg_no);
- lfirst(per_seg_split) = lappend((List *)lfirst(per_seg_split), split);
- }
+ splitToSegmentMaps = lappend(splitToSegmentMaps, NIL);
}
- else
+
+ foreach(lc, splits)
{
- ereport(ERROR,
- (errcode(ERRCODE_CDB_FEATURE_NOT_YET),
- errmsg("Assigning splits to segment without keeping hash distribution policy is not allowed")));
+ int assigned_seg_no;
+ ListCell *per_seg_split;
+ FileSplit split = (FileSplitNode *)lfirst(lc);
+ Assert(split);
+ assigned_seg_no = (split->segno - 1) % target_segment_num;
+ per_seg_split = list_nth_cell(splitToSegmentMaps, assigned_seg_no);
+ lfirst(per_seg_split) = lappend((List *)lfirst(per_seg_split), split);
}
}
else
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/commands/tablecmds.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index c47b828..259821b 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -968,7 +968,12 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt)
locationUris = transformLocationUris(exttypeDesc->location_list,
createExtStmt->formatOpts,
isweb, iswritable);
-
+ int locLength = list_length(exttypeDesc->location_list);
+ if (createStmt->policy && locLength > 0)
+ {
+ createStmt->policy->bucketnum = locLength;
+ }
+
break;
case EXTTBL_TYPE_EXECUTE:
@@ -6437,7 +6442,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap)
tab->relid,
false,
NULL);
- ar_tab->scantable_splits = AssignAOSegFileSplitToSegment(tab->relid, NIL, true,
+ ar_tab->scantable_splits = AssignAOSegFileSplitToSegment(tab->relid, NIL,
target_segment_num, ar_tab->scantable_splits);
/*
* Specify the segno directly as we don't have segno mapping here.
@@ -16719,7 +16724,7 @@ ATPExecPartSplit(Relation rel,
* Dispatch split-related metadata.
*/
scantable_splits = AssignAOSegFileSplitToSegment((Oid)intVal((Value *)pc->partid),
- NIL, true, target_segment_num, scantable_splits);
+ NIL, target_segment_num, scantable_splits);
pc->scantable_splits = scantable_splits;
pc->newpart_aosegnos = segment_segnos;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/parser/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c
index f17c90a..1f48c36 100644
--- a/src/backend/parser/analyze.c
+++ b/src/backend/parser/analyze.c
@@ -2724,7 +2724,7 @@ transformETDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
sizeof(p->attrs[0]));
p->ptype = POLICYTYPE_PARTITIONED;
p->nattrs = 0;
- p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
+ p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetExternalTablePartitionNum());
p->attrs[0] = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/include/access/filesplit.h
----------------------------------------------------------------------
diff --git a/src/include/access/filesplit.h b/src/include/access/filesplit.h
index e075b18..e627b07 100644
--- a/src/include/access/filesplit.h
+++ b/src/include/access/filesplit.h
@@ -65,7 +65,7 @@ typedef SegFileSplitMapNode *SegFileSplitMap;
* table data.
*/
List *
-AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num, List *existings);
+AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num, List *existings);
/*
* Given the relid, and the segment index, return the splits assigned
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 224cb53..21e4c68 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -33,7 +33,6 @@
#include "nodes/parsenodes.h"
#include "executor/execdesc.h"
-#define minimum_segment_num 1
/*
* structure containing information about data residence
* at the host.
[18/50] incubator-hawq git commit: HAWQ-533. Cursor failed,
if don't allocate resource in prepare.
Posted by sh...@apache.org.
HAWQ-533. Cursor failed, if don't allocate resource in prepare.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/96345af1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/96345af1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/96345af1
Branch: refs/heads/HAWQ-459
Commit: 96345af194bfd3ca05227037479c9835a5e495b0
Parents: 5fda182
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 10:59:38 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 11:02:25 2016 +0800
----------------------------------------------------------------------
src/backend/executor/spi.c | 24 +++++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96345af1/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 8e7645c..9465de8 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1146,6 +1146,20 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
qtlist = copyObject(qtlist);
ptlist = copyObject(ptlist);
+ PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
+
+ if ( (Gp_role == GP_ROLE_DISPATCH) &&
+ (stmt->resource_parameters != NULL) )
+ {
+ /*
+ * Now, we want to allocate resource.
+ */
+ stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
+ stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
+ stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
+ stmt->resource_parameters->vol_info_size);
+ }
+
/* If the plan has parameters, set them up */
if (spiplan->nargs > 0)
{
@@ -1846,12 +1860,12 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
* We only allocate resource for multiple executions of queries, NOT for utility commands.
* SELECT/INSERT are supported at present.
*/
- if( (queryTree->commandType == CMD_SELECT) ||
- (queryTree->commandType == CMD_INSERT) )
+ if((queryTree->commandType == CMD_SELECT) ||
+ (queryTree->commandType == CMD_INSERT))
{
- if ( (Gp_role == GP_ROLE_DISPATCH) &&
- (stmt->resource == NULL) &&
- (stmt->resource_parameters != NULL) )
+ if ((Gp_role == GP_ROLE_DISPATCH) &&
+ (stmt->resource == NULL) &&
+ (stmt->resource_parameters != NULL))
{
SplitAllocResult *allocResult = NULL;
[31/50] incubator-hawq git commit: HAWQ-555. Remove unused guc for
allocating resource for tolerating variance of number of virtual segments
Posted by sh...@apache.org.
HAWQ-555. Remove unused guc for allocating resource for tolerating variance of number of virtual segments
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2c033d9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2c033d9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2c033d9f
Branch: refs/heads/HAWQ-459
Commit: 2c033d9f069bb2fe092dba3ec4ed9cb2d2e13e5f
Parents: 3ae24a3
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:41:49 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:41:49 2016 +1100
----------------------------------------------------------------------
src/backend/cdb/cdbvars.c | 2 --
src/backend/resourcemanager/resourcepool.c | 4 ++--
src/backend/utils/misc/guc.c | 11 -----------
3 files changed, 2 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index b4a9f3d..a2476fd 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -390,8 +390,6 @@ double rm_regularize_io_factor;
double rm_regularize_usage_factor;
double rm_regularize_nvseg_factor;
-int rm_nvseg_variance_among_seg_respool_limit;
-
/* Greenplum Database Experimental Feature GUCs */
int gp_distinct_grouping_sets_threshold = 32;
bool gp_enable_explain_allstat = FALSE;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index ca275f5..e5d083c 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2556,7 +2556,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
Assert(minnvseg <= maxnvseg);
- if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_respool_limit )
+ if ( maxnvseg - minnvseg > rm_tolerate_nseg_limit )
{
elog(LOG, "Reject virtual segment allocation based on data "
"locality information. After tentative allocation "
@@ -2565,7 +2565,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
"is %d, tolerated difference limit is %d.",
maxnvseg,
minnvseg,
- rm_nvseg_variance_among_seg_respool_limit);
+ rm_tolerate_nseg_limit);
/* Return the allocated resource. */
List *vsegcntlist = NULL;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 54ef13f..3d13eb4 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6557,17 +6557,6 @@ static struct config_int ConfigureNamesInt[] =
},
{
- {"hawq_rm_nvseg_variance_amon_seg_respool_limit", PGC_POSTMASTER, RESOURCES_MGM,
- gettext_noop("the variance of vseg number in each segment that resource manager "
- "should tolerate at most in resource pool when choosing segments "
- "based on data locality reference."),
- NULL
- },
- &rm_nvseg_variance_among_seg_respool_limit,
- 2, 0, 65535, NULL, NULL
- },
-
- {
{"hawq_rm_container_batch_limit", PGC_POSTMASTER, RESOURCES_MGM,
gettext_noop("the batch process limit for global resource manager containers."),
NULL
[44/50] incubator-hawq git commit: HAWQ-502 drop boost-thread link
flag [#105338218]
Posted by sh...@apache.org.
HAWQ-502 drop boost-thread link flag [#105338218]
Because it's unnecessary.
This fixes #420
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6e88cbab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6e88cbab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6e88cbab
Branch: refs/heads/HAWQ-459
Commit: 6e88cbabfe985bbb3f38f32a821ae84158735419
Parents: c67a1a4
Author: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Authored: Fri Mar 18 17:01:38 2016 -0700
Committer: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <xz...@pivotal.io>
Committed: Fri Mar 18 17:03:40 2016 -0700
----------------------------------------------------------------------
src/backend/Makefile | 2 +-
src/bin/gpcheckhdfs/Makefile | 2 +-
src/bin/gpfilesystem/hdfs/Makefile | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 18ed91a..f89c24d 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,7 +52,7 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
# The backend doesn't need everything that's in LIBS, however
LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
# adding orca libraries
ifeq ($(enable_orca),yes)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index cd81902..fd3c4f1 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
OBJS=gpcheckhdfs.o
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index b0fb359..cfe1995 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS = gpfshdfs.o
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3
ifdef USE_PGXS
PGXS := $(shell pg_config --pgxs)
[23/50] incubator-hawq git commit: Revert "HAWQ-529. Allocate
resource for udf in resource negotiator."
Posted by sh...@apache.org.
Revert "HAWQ-529. Allocate resource for udf in resource negotiator."
This reverts commit 8e38e844f0d2a1c45f7d5cd25a8f5b639c8b6fb7.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fd202277
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fd202277
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fd202277
Branch: refs/heads/HAWQ-459
Commit: fd2022772d1f39e5f205347a4c0cfd0a0f04021f
Parents: bc065a5
Author: hzhang2 <zh...@163.com>
Authored: Thu Mar 17 09:19:04 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Thu Mar 17 09:19:04 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdatalocality.c | 194 ++++++++++--------------------
src/backend/executor/spi.c | 72 ++---------
src/backend/optimizer/plan/planner.c | 57 +--------
src/backend/optimizer/util/clauses.c | 11 +-
src/backend/optimizer/util/walkers.c | 1 -
src/include/cdb/cdbdatalocality.h | 19 +--
src/include/executor/spi.h | 7 --
src/include/optimizer/planner.h | 4 -
8 files changed, 76 insertions(+), 289 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index d15d6b0..c1f829b 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,9 +33,6 @@
#include "access/filesplit.h"
#include "access/parquetsegfiles.h"
#include "catalog/catalog.h"
-#include "catalog/catquery.h"
-#include "catalog/pg_inherits.h"
-#include "catalog/pg_proc.h"
#include "cdb/cdbdatalocality.h"
#include "cdb/cdbutil.h"
#include "cdb/cdbvars.h"
@@ -44,7 +41,6 @@
#include "utils/tqual.h"
#include "utils/memutils.h"
#include "executor/execdesc.h"
-#include "executor/spi.h"
#include "nodes/nodes.h"
#include "nodes/parsenodes.h"
#include "optimizer/walkers.h"
@@ -58,7 +54,11 @@
#include "catalog/pg_proc.h"
#include "postgres.h"
#include "resourcemanager/utils/hashtable.h"
+#include "catalog/pg_inherits.h"
+//#include "utils/misc/guc.c"
+#define PRONAME 1
+#define PROISAGG 5
/* We need to build a mapping from host name to host index */
extern bool optimizer; /* Enable the optimizer */
@@ -372,7 +372,7 @@ static Block_Host_Index * update_data_dist_stat(
static HostDataVolumeInfo *search_host_in_stat_context(
split_to_segment_mapping_context *context, char *hostname);
-static bool IsBuildInFunction(Oid funcOid);
+static bool IsAggFunction(char* funcName);
static bool allocate_hash_relation(Relation_Data* rel_data,
Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -630,36 +630,39 @@ static void collect_range_tables(Query *query, List* full_range_table,
/*
*
*/
-static bool IsBuildInFunction(Oid foid) {
+static bool IsAggFunction(char* funcName) {
+ if (funcName == NULL) {
+ return false;
+ }
+ Relation pg_proc_rel;
+ TupleDesc pg_proc_dsc;
+ HeapTuple tuple;
+ SysScanDesc pg_proc_scan;
- cqContext *pcqCtx;
- HeapTuple readtup = NULL;
- HeapTuple procedureTuple;
- Form_pg_proc procedureStruct;
+ pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
+ pg_proc_dsc = RelationGetDescr(pg_proc_rel);
+ ScanKeyData skey;
- /*
- * get the procedure tuple corresponding to the given function Oid
- */
- pcqCtx = caql_beginscan(
- NULL,
- cql("SELECT * FROM pg_proc "
- " WHERE oid = :1 ",
- ObjectIdGetDatum(foid)));
-
- procedureTuple = caql_getnext(pcqCtx);
-
- if (!HeapTupleIsValid(procedureTuple))
- elog(ERROR, "cache lookup failed for function %u", foid);
- procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
- caql_endscan(pcqCtx);
- /* we treat proc namespace = 11 to build in function.*/
- if (procedureStruct->pronamespace == 11) {
- return true;
- } else {
- return false;
+ ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
+ F_NAMEEQ, CStringGetDatum(funcName));
+
+ pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
+ ActiveSnapshot, 1, &skey);
+ while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+
+ bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
+ systable_endscan(pg_proc_scan);
+ heap_close(pg_proc_rel, AccessShareLock);
+ if (isAgg) {
+ return true;
+ } else {
+ return false;
+ }
}
+ systable_endscan(pg_proc_scan);
+ heap_close(pg_proc_rel, AccessShareLock);
+ return true;
}
-
/*
*
*/
@@ -673,6 +676,25 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
foreach(old_lc, *range_tables)
{
RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
+ if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
+ *isTableFunctionExists = true;
+ }
+ if (entry->rtekind == RTE_SUBQUERY) {
+ Query* subQuery = entry->subquery;
+ ListCell *lc;
+ foreach(lc, subQuery->targetList)
+ {
+ TargetEntry *te = (TargetEntry *) lfirst(lc);
+ bool isAggFunc = IsAggFunction(te->resname);
+ // if target list of subquery contains non aggregate function,
+ // then we consider the query contains and use default_segment_num guc
+ // as the number of virtual segment
+ if (!isAggFunc) {
+ *isTableFunctionExists = true;
+ }
+ }
+
+ }
if (entry->rtekind != RTE_RELATION) {
continue;
}
@@ -3936,54 +3958,11 @@ static void cleanup_allocation_algorithm(
}
/*
- * udf_collector_walker: the routine to file udfs.
- */
-bool udf_collector_walker(Node *node,
- udf_collector_context *context) {
- if (node == NULL) {
- return false;
- }
-
- if (IsA(node, Query)) {
- return query_tree_walker((Query *) node, udf_collector_walker,
- (void *) context,
- QTW_EXAMINE_RTES);
- }
-
- /*For Aggref, we don't consider it as udf.*/
-
- if(IsA(node,FuncExpr)){
- if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
- context->udf_exist = true;
- }
- return false;
- }
-
- return expression_tree_walker(node, udf_collector_walker,
- (void *) context);
-
- return false;
-}
-
-/*
- * find_udf: collect all udf, and store them into the udf_collector_context.
- */
-void find_udf(Query *query, udf_collector_context *context) {
-
- query_tree_walker(query, udf_collector_walker, (void *) context,
- QTW_EXAMINE_RTES);
-
- return;
-}
-
-
-/*
* calculate_planner_segment_num
- * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
*/
SplitAllocResult *
calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
+ List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
SplitAllocResult *result = NULL;
QueryResource *resource = NULL;
QueryResourceParameters *resource_parameters = NULL;
@@ -3992,14 +3971,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
List *alloc_result = NIL;
split_to_segment_mapping_context context;
- int planner_segments = 0; /*virtual segments number for explain statement */
+ int planner_segments = -1; /*virtual segments number for explain statement */
result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
result->resource = NULL;
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = 0;
+ result->planner_segments = -1;
result->datalocalityInfo = makeStringInfo();
/* fake data locality */
@@ -4016,7 +3995,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = 0;
+ result->planner_segments = -1;
return result;
}
@@ -4041,11 +4020,6 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
* 5 data size of random "from" relation
*/
- udf_collector_context udf_context;
- udf_context.udf_exist = false;
-
- find_udf(query, &udf_context);
- isTableFunctionExists = udf_context.udf_exist;
/*convert range table list to oid list and check whether table function exists
*we keep a full range table list and a range table list without result relation separately
*/
@@ -4072,15 +4046,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
/*use inherit resource*/
if (resourceLife == QRL_INHERIT) {
-
- if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
- {
- resource = NULL;
- }
- else
- {
- resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
- }
+ resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4231,11 +4197,6 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
maxTargetSegmentNumber = enforce_virtual_segment_number;
minTargetSegmentNumber = enforce_virtual_segment_number;
}
- /* in PBE mode, the execute should use the same vseg number. */
- if(fixedVsegNum > 0 ){
- maxTargetSegmentNumber = fixedVsegNum;
- minTargetSegmentNumber = fixedVsegNum;
- }
uint64_t before_rm_allocate_resource = gettime_microsec();
/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4243,40 +4204,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
mincost <<= 20;
int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
if (QRL_NONE != resourceLife) {
-
- if (SPI_IsInPrepare())
- {
- resource = NULL;
- /*
- * prepare need to get resource quota from RM
- * and pass quota(planner_segments) to Orca or Planner to generate plan
- * the following executes(in PBE) should reallocate the same number
- * of resources.
- */
- uint32 seg_num;
- uint32 seg_num_min;
- uint32 seg_memory_mb;
- double seg_core;
-
- GetResourceQuota(maxTargetSegmentNumber,
- minTargetSegmentNumber,
- &seg_num,
- &seg_num_min,
- &seg_memory_mb,
- &seg_core);
-
- planner_segments = seg_num;
- minTargetSegmentNumber = planner_segments;
- maxTargetSegmentNumber = planner_segments;
- }
- else
- {
- resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
- maxTargetSegmentNumber,
- minTargetSegmentNumber,
- context.host_context.hostnameVolInfos,
- context.host_context.size);
- }
+ resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+ maxTargetSegmentNumber, minTargetSegmentNumber,
+ context.host_context.hostnameVolInfos, context.host_context.size);
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4306,7 +4236,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
if (resource == NULL) {
result->resource = NULL;
- result->resource_parameters = resource_parameters;
+ result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
result->planner_segments = planner_segments;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 8e7645c..07812aa 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,7 +46,6 @@
#include "cdb/memquota.h"
#include "executor/nodeFunctionscan.h"
#include "nodes/stack.h"
-#include "cdb/cdbdatalocality.h"
extern char *savedSeqServerHost;
extern int savedSeqServerPort;
@@ -75,7 +74,6 @@ static int _SPI_curid = -1;
static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
static char *_QD_currently_prepared_stmt = NULL;
-static int SPI_prepare_depth = 0;
static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
@@ -108,32 +106,6 @@ static bool _SPI_checktuples(void);
/* =================== interface functions =================== */
-bool SPI_IsInPrepare(void)
-{
- if (SPI_prepare_depth > 0)
- {
- return true;
- }
- else if (SPI_prepare_depth < 0)
- {
- elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
- SPI_prepare_depth);
- }
-
- return false;
-}
-
-void SPI_IncreasePrepareDepth(void)
-{
- SPI_prepare_depth++;
-}
-
-void SPI_DecreasePrepareDepth(void)
-{
- SPI_prepare_depth--;
-}
-
-
int
SPI_connect(void)
{
@@ -594,8 +566,6 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
_SPI_plan plan;
_SPI_plan *result;
- SPI_IncreasePrepareDepth();
-
if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
{
SPI_result = SPI_ERROR_ARGUMENT;
@@ -621,13 +591,9 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
/* copy plan to procedure context */
result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
-
- SPI_DecreasePrepareDepth();
}
PG_CATCH();
{
- SPI_DecreasePrepareDepth();
-
_SPI_end_call(true);
PG_RE_THROW();
}
@@ -1853,37 +1819,13 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
(stmt->resource == NULL) &&
(stmt->resource_parameters != NULL) )
{
- SplitAllocResult *allocResult = NULL;
-
- /* If this is a parallel plan. */
- if (stmt->planTree->dispatch == DISPATCH_PARALLEL)
- {
- /*
- * Now, we want to allocate resource.
- */
- allocResult = calculate_planner_segment_num(queryTree,
- stmt->resource_parameters->life,
- stmt->rtable,
- stmt->intoPolicy,
- stmt->nMotionNodes + stmt->nInitPlans + 1,
- stmt->resource_parameters->min_target_segment_num);
-
- Assert(allocResult);
-
- if(stmt->resource !=NULL)
- {
- pfree(stmt->resource);
- }
- stmt->resource = allocResult->resource;
- if(stmt->scantable_splits !=NULL)
- {
- list_free_deep(stmt->scantable_splits);
- }
- stmt->scantable_splits = allocResult->alloc_results;
- stmt->planner_segments = allocResult->planner_segments;
- stmt->datalocalityInfo = allocResult->datalocalityInfo;
- pfree(allocResult);
- }
+ stmt->resource = AllocateResource(stmt->resource_parameters->life,
+ stmt->resource_parameters->slice_size,
+ stmt->resource_parameters->iobytes,
+ stmt->resource_parameters->max_target_segment_num,
+ stmt->resource_parameters->min_target_segment_num,
+ stmt->resource_parameters->vol_info,
+ stmt->resource_parameters->vol_info_size);
}
originalStmt->resource = NULL;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index c6be59a..be057e0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,8 +91,6 @@ planner_hook_type planner_hook = NULL;
ParamListInfo PlannerBoundParamList = NULL; /* current boundParams */
-static int PlanningDepth = 0; /* Planning depth */
-
/* Expression kind codes for preprocess_expression */
#define EXPRKIND_QUAL 0
#define EXPRKIND_TARGET 1
@@ -167,32 +165,6 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
bool is_dummy_plan(Plan *plan);
-
-bool is_in_planning_phase(void)
-{
- if (PlanningDepth > 0)
- {
- return true;
- }
- else if (PlanningDepth < 0)
- {
- elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
- }
-
- return false;
-}
-
-void increase_planning_depth(void)
-{
- PlanningDepth++;
-}
-
-void decrease_planning_depth(void)
-{
- PlanningDepth--;
-}
-
-
#ifdef USE_ORCA
/**
* Logging of optimization outcome
@@ -313,7 +285,7 @@ planner(Query *parse, int cursorOptions,
PlannedStmt *result = NULL;
instr_time starttime, endtime;
ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
- SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
+ SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
ppResult->saResult = initResult;
ppResult->stmt = NULL;
static int plannerLevel = 0;
@@ -328,8 +300,6 @@ planner(Query *parse, int cursorOptions,
* resource to run this query. After gaining the resource, we can perform the
* actual optimization.
*/
- increase_planning_depth();
-
plannerLevel++;
if (!resourceNegotiateDone)
{
@@ -339,8 +309,6 @@ planner(Query *parse, int cursorOptions,
{
resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
- decrease_planning_depth();
-
if(ppResult->stmt && ppResult->stmt->planTree)
{
isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -350,8 +318,6 @@ planner(Query *parse, int cursorOptions,
}
PG_CATCH();
{
- decrease_planning_depth();
-
if ((ppResult != NULL))
{
pfree(ppResult);
@@ -464,7 +430,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = 0;
+ gp_segments_for_planner = -1;
}
SetActiveQueryResource(savedQueryResource);
if ((ppResult != NULL))
@@ -484,7 +450,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = 0;
+ gp_segments_for_planner = -1;
}
SetActiveQueryResource(savedQueryResource);
@@ -504,7 +470,6 @@ planner(Query *parse, int cursorOptions,
return result;
}
-
/*
* The new framework for HAWQ 2.0 query optimizer
*/
@@ -515,8 +480,6 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
PlannedStmt *plannedstmt = NULL;
do
{
- udf_collector_context udf_context;
- udf_context.udf_exist = false;
SplitAllocResult *allocResult = NULL;
Query *my_parse = copyObject(parse);
ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -532,24 +495,12 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
*/
allocResult = calculate_planner_segment_num(my_parse, resourceLife,
plannedstmt->rtable, plannedstmt->intoPolicy,
- plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
- -1);
+ plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
Assert(allocResult);
(*result)->saResult = *allocResult;
pfree(allocResult);
- }else{
- find_udf(my_parse, &udf_context);
- if(udf_context.udf_exist){
- if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
- int64 mincost = min_cost_for_each_query;
- mincost <<= 20;
- int avgSliceNum = 3;
- (*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
- GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
- }
- }
}
} while (0);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a3d67f2..a80737f 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,15 +2831,8 @@ simplify_function(Oid funcid, Oid result_type, List *args,
if (!HeapTupleIsValid(func_tuple))
elog(ERROR, "cache lookup failed for function %u", funcid);
- if (is_in_planning_phase())
- {
- newexpr = NULL;
- }
- else
- {
- newexpr = evaluate_function(funcid, result_type, args,
- func_tuple, context);
- }
+ newexpr = evaluate_function(funcid, result_type, args,
+ func_tuple, context);
if (large_const(newexpr, context->max_size))
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index a8222fb..6f1ee51 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,7 +162,6 @@ expression_tree_walker(Node *node,
case T_PartBoundExpr:
case T_PartBoundInclusionExpr:
case T_PartBoundOpenExpr:
- case T_RangeTblEntry :
/* primitive node types with no expression subnodes */
break;
case T_Aggref:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 67375d7..95f2bd5 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -49,13 +49,6 @@ typedef struct SplitAllocResult
} SplitAllocResult;
/*
- * structure containing all relation range table entries.
- */
-typedef struct udf_collector_context {
- bool udf_exist;
-} udf_collector_context;
-
-/*
* structure containing rel and type when execution
*/
typedef struct CurrentRelType {
@@ -93,17 +86,7 @@ void saveQueryResourceParameters(
* we calculate the appropriate planner segment_num.
*/
SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
-
-/*
- * udf_collector_walker: the routine to file udfs.
- */
-bool udf_collector_walker(Node *node, udf_collector_context *context);
-
-/*
- * find_udf: collect all udf, and store them into the udf_collector_context.
- */
-void find_udf(Query *query, udf_collector_context *context);
+ List *rtable, GpPolicy *intoPolicy, int sliceNum);
FILE *fp;
FILE *fpaoseg;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 7b4ba1b..867daf5 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,11 +155,4 @@ extern uint64 SPI_GetMemoryReservation(void);
extern void SPI_ReserveMemory(uint64 mem_reserved);
extern bool SPI_IsMemoryReserved(void);
-/**
- * Query resource related routines.
- */
-extern bool SPI_IsInPrepare(void);
-extern void SPI_IncreasePrepareCounter(void);
-extern void SPI_DecreasePrepareCounter(void);
-
#endif /* SPI_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index cc2eb7d..48de348 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,8 +50,4 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
double dNumGroups,
AggClauseCounts *agg_counts);
-extern bool is_in_planning_phase(void);
-extern void increase_planning_depth(void);
-extern void decrease_planning_depth(void);
-
#endif /* PLANNER_H */
[05/50] incubator-hawq git commit: HAWQ-524. Modify processing for
cdbDispatchResult->resultBuf when creation and free
Posted by sh...@apache.org.
HAWQ-524. Modify processing for cdbDispatchResult->resultBuf when creation and free
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ac26f48f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ac26f48f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ac26f48f
Branch: refs/heads/HAWQ-459
Commit: ac26f48fd44af17c26cee0ce48a44bf50ddeb9ca
Parents: 4b53168
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 17:47:42 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Tue Mar 15 09:38:40 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdispatchresult.c | 19 +++++++++++--------
src/backend/cdb/executormgr.c | 4 +++-
2 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/cdbdispatchresult.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdispatchresult.c b/src/backend/cdb/cdbdispatchresult.c
index b036b26..9ba90c0 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -183,15 +183,18 @@ void
cdbdisp_resetResult(CdbDispatchResult *dispatchResult)
{
PQExpBuffer buf = dispatchResult->resultbuf;
- PGresult **begp = (PGresult **)buf->data;
- PGresult **endp = (PGresult **)(buf->data + buf->len);
- PGresult **p;
-
- /* Free the PGresult objects. */
- for (p = begp; p < endp; ++p)
+ if (buf != NULL)
{
- Assert(*p != NULL);
- PQclear(*p);
+ PGresult **begp = (PGresult **)buf->data;
+ PGresult **endp = (PGresult **)(buf->data + buf->len);
+ PGresult **p;
+
+ /* Free the PGresult objects. */
+ for (p = begp; p < endp; ++p)
+ {
+ Assert(*p != NULL);
+ PQclear(*p);
+ }
}
/* Reset summary indicators. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 88dd32f..47f5bbe 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -220,7 +220,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
executor->refResult = cdbdisp_makeResult(dispatch_get_results(data),
executor->desc,
dispatch_get_task_identity(task)->slice_id);
- Assert(executor->refResult != NULL);
+
+ if (executor->refResult == NULL)
+ return false;
/* Transfer any connection errors from segdbDesc. */
executormgr_merge_error(executor);
[24/50] incubator-hawq git commit: HAWQ-549. Ignore empty 'pg_log'
folder while doing standby data directory check
Posted by sh...@apache.org.
HAWQ-549. Ignore empty 'pg_log' folder while doing standby data directory check
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aef7bd2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aef7bd2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aef7bd2f
Branch: refs/heads/HAWQ-459
Commit: aef7bd2f3465684d84b7b9ee515aee39f4346780
Parents: fd20227
Author: rlei <rl...@pivotal.io>
Authored: Wed Mar 16 16:53:12 2016 +0800
Committer: rlei <rl...@pivotal.io>
Committed: Thu Mar 17 10:36:09 2016 +0800
----------------------------------------------------------------------
tools/bin/lib/hawqinit.sh | 40 ++++++++++++++++++++++++++++++++++++++--
1 file changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aef7bd2f/tools/bin/lib/hawqinit.sh
----------------------------------------------------------------------
diff --git a/tools/bin/lib/hawqinit.sh b/tools/bin/lib/hawqinit.sh
index dab0b34..43ffdef 100755
--- a/tools/bin/lib/hawqinit.sh
+++ b/tools/bin/lib/hawqinit.sh
@@ -473,8 +473,13 @@ check_data_directorytory() {
${MKDIR} -p ${default_sdd}
fi
# Check if data directory already exist and clean.
+ if [ "${hawq_data_directory}" = "" ]; then
+ LOG_MSG "[ERROR]:-Data directory path is not valid value on ${host_name}" verbose
+ exit 1
+ fi
+
if [ -d ${hawq_data_directory} ]; then
- if [ "$(ls -A ${hawq_data_directory})" ] && [ "${hawq_data_directory}" != "" ]; then
+ if [ "$(ls -A ${hawq_data_directory})" ]; then
LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
exit 1
fi
@@ -484,6 +489,37 @@ check_data_directorytory() {
fi
}
+check_standby_data_directorytory() {
+ # If it's default directory, create it if not exist.
+ default_mdd=~/hawq-data-directory/masterdd
+ default_sdd=~/hawq-data-directory/segmentdd
+ if [ "${hawq_data_directory}" = "${default_mdd}" ]; then
+ ${MKDIR} -p ${default_mdd}
+ elif [ "${hawq_data_directory}" = "${default_sdd}" ]; then
+ ${MKDIR} -p ${default_sdd}
+ fi
+ # Check if data directory already exist and clean.
+ if [ "${hawq_data_directory}" = "" ]; then
+ LOG_MSG "[ERROR]:-Data directory path is not valid value on ${host_name}" verbose
+ exit 1
+ fi
+
+ if [ -d "${hawq_data_directory}" ]; then
+ if [ "$(ls -A ${hawq_data_directory}| ${GREP} -v pg_log)" ]; then
+ LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
+ exit 1
+ else
+ if [ -d ${hawq_data_directory}/pg_log ] && [ "$(ls -A ${hawq_data_directory}/pg_log)" ]; then
+ LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
+ exit 1
+ fi
+ fi
+ else
+ LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} does not exist on ${host_name}, please create it" verbose
+ exit 1
+ fi
+}
+
check_temp_directory() {
# Check if temp directory exist.
for tmp_dir in ${tmp_dir_list}; do
@@ -504,7 +540,7 @@ if [ ${object_type} == "master" ]; then
check_temp_directory
master_init
elif [ ${object_type} == "standby" ]; then
- check_data_directorytory
+ check_standby_data_directorytory
standby_init
elif [ ${object_type} == "segment" ]; then
check_data_directorytory
[07/50] incubator-hawq git commit: HAWQ-404. Add sort during INSERT
of append only AO partition tables
Posted by sh...@apache.org.
HAWQ-404. Add sort during INSERT of append only AO partition tables
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2a150660
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2a150660
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2a150660
Branch: refs/heads/HAWQ-459
Commit: 2a150660417bd61db1d0b30c7e7bce9222e2fda1
Parents: ac26f48 6e6890a
Author: ivan <iw...@pivotal.io>
Authored: Tue Mar 15 09:53:24 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Tue Mar 15 09:53:24 2016 +0800
----------------------------------------------------------------------
src/backend/executor/execDML.c | 99 +++++++++------
src/backend/executor/execMain.c | 13 +-
src/backend/executor/execUtils.c | 2 +-
src/backend/executor/nodeDML.c | 2 +-
src/backend/gpopt/ivy.xml | 2 +-
.../gpopt/translate/CTranslatorDXLToPlStmt.cpp | 2 +
.../translate/CTranslatorRelcacheToDXL.cpp | 9 ++
src/backend/gpopt/utils/COptTasks.cpp | 4 +-
src/backend/nodes/copyfuncs.c | 1 +
src/backend/nodes/outfast.c | 1 +
src/backend/nodes/outfuncs.c | 1 +
src/backend/nodes/readfast.c | 1 +
src/backend/utils/misc/guc.c | 11 ++
src/include/executor/execDML.h | 3 +-
src/include/gpopt/translate/CTranslatorUtils.h | 4 -
src/include/nodes/execnodes.h | 7 +-
src/include/nodes/plannodes.h | 1 +
src/include/utils/guc.h | 1 +
src/test/regress/expected/goh_partition.out | 126 +++++++++++++++++++
.../expected/gpsql_alter_table_optimizer.out | 2 +-
src/test/regress/expected/insert_optimizer.out | 2 +-
src/test/regress/sql/goh_partition.sql | 68 ++++++++++
22 files changed, 304 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a150660/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a150660/src/include/utils/guc.h
----------------------------------------------------------------------
[20/50] incubator-hawq git commit: HAWQ-539. Add fault injection for
dispatcher QD site
Posted by sh...@apache.org.
HAWQ-539. Add fault injection for dispatcher QD site
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1a75a2a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1a75a2a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1a75a2a7
Branch: refs/heads/HAWQ-459
Commit: 1a75a2a7c191ce16371d9e41d62786228dfa5774
Parents: 4d1a854
Author: Chunling Wang <wa...@ChunlingdeMacBook-Pro.local>
Authored: Tue Mar 15 10:15:03 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Wed Mar 16 14:09:11 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdispatchresult.c | 9 +++++++++
src/backend/cdb/dispatcher.c | 9 +++++++++
src/backend/cdb/workermgr.c | 10 ++++++++++
src/backend/utils/misc/faultinjector.c | 9 ++++++++-
src/include/utils/faultinjector.h | 2 ++
tools/bin/gppylib/programs/clsInjectFault.py | 5 ++++-
tools/bin/hawqpylib/programs/clsInjectFault.py | 5 ++++-
7 files changed, 46 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/cdbdispatchresult.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdispatchresult.c b/src/backend/cdb/cdbdispatchresult.c
index 9ba90c0..d97dda5 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -33,6 +33,7 @@
#include "lib/stringinfo.h" /* StringInfoData */
#include "utils/guc.h" /* log_min_messages */
+#include "utils/faultinjector.h"
#include "cdb/cdbconn.h" /* SegmentDatabaseDescriptor */
#include "cdb/cdbpartition.h"
@@ -110,6 +111,14 @@ cdbdisp_makeResult(struct CdbDispatchResults *meleeResults,
meleeIndex = meleeResults->resultCount++;
dispatchResult = &meleeResults->resultArray[meleeIndex];
+#ifdef FAULT_INJECTOR
+ FaultInjector_InjectFaultIfSet(
+ CreateCdbDispathResultObject,
+ DDLNotSpecified,
+ "", // databaseName
+ ""); // tableName
+#endif
+
/* Initialize CdbDispatchResult. */
dispatchResult->meleeResults = meleeResults;
dispatchResult->meleeIndex = meleeIndex;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 20e97ab..5d7105c 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -49,6 +49,7 @@
#include "utils/memutils.h" /* GetMemoryChunkContext */
#include "cdb/cdbsrlz.h" /* serializeNode */
#include "utils/datum.h" /* datumGetSize */
+#include "utils/faultinjector.h"
#include "utils/lsyscache.h" /* get_typlenbyval */
#include "miscadmin.h" /* CHECK_FOR_INTERRUPTS */
#include "tcop/tcopprot.h" /* ResetUsage */
@@ -963,6 +964,14 @@ dispatcher_bind_executor(DispatchData *data)
continue;
}
+#ifdef FAULT_INJECTOR
+ FaultInjector_InjectFaultIfSet(
+ ConnectionFailAfterGangCreation,
+ DDLNotSpecified,
+ "", // databaseName
+ ""); // tableName
+#endif
+
if (!executormgr_bind_executor_task(data, executor, desc, task, slice))
return false;
data->num_of_cached_executors++;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/workermgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/workermgr.c b/src/backend/cdb/workermgr.c
index eaf3060..2a31d29 100644
--- a/src/backend/cdb/workermgr.c
+++ b/src/backend/cdb/workermgr.c
@@ -35,6 +35,8 @@
#include "cdb/cdbgang.h" /* gp_pthread_create */
#include "miscadmin.h" /* TODO: InterruptPending */
+#include "utils/faultinjector.h"
+
/*
* This structure abstract the general job.
@@ -114,6 +116,14 @@ workermgr_submit_job(WorkerMgrState *state,
i++;
worker_mgr_thread->func = func;
+#ifdef FAULT_INJECTOR
+ FaultInjector_InjectFaultIfSet(
+ WorkerManagerSubmitJob,
+ DDLNotSpecified,
+ "", // databaseName
+ ""); // tableName
+#endif
+
worker_mgr_thread->thread_ret = gp_pthread_create(&worker_mgr_thread->thread, workermgr_thread_func, worker_mgr_thread, "submit_plan_to_qe");
if (worker_mgr_thread->thread_ret)
goto error_cleanup;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/utils/misc/faultinjector.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/faultinjector.c b/src/backend/utils/misc/faultinjector.c
index 44b7462..f5ab5bc 100644
--- a/src/backend/utils/misc/faultinjector.c
+++ b/src/backend/utils/misc/faultinjector.c
@@ -281,8 +281,12 @@ FaultInjectorIdentifierEnumToString[] = {
/* inject fault after dispatcher thread creation*/
/* _("dispatch_wait"),
inject fault after dispatcher wait for results from segments*/
- _("connection_fail_after_gang"),
+ _("connection_fail_after_gang_creation"),
/* inject fault after gang thread creation, set connection null*/
+ _("create_cdb_dispath_result_object"),
+ /* inject fault when create cdb dispatch result object, set out of memory */
+ _("worker_manager_submit_job"),
+ /* inject fault when worker manager submit job , set error*/
/* _("make_dispatch_thread"),
inject fault when initialing memory structure for dispatcher thread*/
_("before_dispatch"),
@@ -1070,6 +1074,9 @@ FaultInjector_NewHashEntry(
case DtmXLogDistributedCommit:
case AnalyzeSubxactError:
case OptTaskAllocateStringBuffer:
+ case ConnectionFailAfterGangCreation:
+ case CreateCdbDispathResultObject:
+ case WorkerManagerSubmitJob:
/* These faults are designed for master. */
if(!AmIMaster())
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/include/utils/faultinjector.h
----------------------------------------------------------------------
diff --git a/src/include/utils/faultinjector.h b/src/include/utils/faultinjector.h
index f197d2e..ca9c278 100644
--- a/src/include/utils/faultinjector.h
+++ b/src/include/utils/faultinjector.h
@@ -188,6 +188,8 @@ typedef enum FaultInjectorIdentifier_e {
ConnectionFailAfterGangCreation,
/* MakeDispatchThread,*/
+ CreateCdbDispathResultObject,
+ WorkerManagerSubmitJob,
BeforeDispatch,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/gppylib/programs/clsInjectFault.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/programs/clsInjectFault.py b/tools/bin/gppylib/programs/clsInjectFault.py
index be6d5ce..6f097d8 100644
--- a/tools/bin/gppylib/programs/clsInjectFault.py
+++ b/tools/bin/gppylib/programs/clsInjectFault.py
@@ -405,7 +405,10 @@ class GpInjectFaultProgram:
"analyze_subxact_error (inject an error during analyze)," \
"opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \
"runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \
- "all (affects all faults injected, used for 'status' and 'reset'), ")
+ "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \
+ "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memeory)" \
+ "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \
+ "all (affects all faults injected, used for 'status' and 'reset'), ")
addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string",
metavar="ddlStatement",
help="The DDL statement on which fault should be set and triggered " \
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/hawqpylib/programs/clsInjectFault.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/programs/clsInjectFault.py b/tools/bin/hawqpylib/programs/clsInjectFault.py
index af4d9df..8992f9d 100644
--- a/tools/bin/hawqpylib/programs/clsInjectFault.py
+++ b/tools/bin/hawqpylib/programs/clsInjectFault.py
@@ -413,7 +413,10 @@ class HAWQInjectFaultProgram:
"analyze_subxact_error (inject an error during analyze)," \
"opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \
"runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \
- "all (affects all faults injected, used for 'status' and 'reset'), ")
+ "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \
+ "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memory)" \
+ "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \
+ "all (affects all faults injected, used for 'status' and 'reset'), ")
addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string",
metavar="ddlStatement",
help="The DDL statement on which fault should be set and triggered " \
[36/50] incubator-hawq git commit: HAWQ-532. Optimise vseg number for
copy to statement.
Posted by sh...@apache.org.
HAWQ-532. Optimise vseg number for copy to statement.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c29d6476
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c29d6476
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c29d6476
Branch: refs/heads/HAWQ-459
Commit: c29d647658e5e4219a370b4869f68cdcd577ab65
Parents: eaaf945
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:30:20 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbmutate.c | 2 +-
src/backend/commands/copy.c | 96 ++++++++++++++++++++++++++++---
src/backend/commands/prepare.c | 2 +-
src/backend/optimizer/util/relnode.c | 2 +-
src/backend/parser/analyze.c | 8 +--
src/backend/postmaster/identity.c | 2 +-
src/backend/utils/misc/guc.c | 12 ++--
src/include/postmaster/identity.h | 2 +-
8 files changed, 103 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/cdb/cdbmutate.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c
index 6959c64..9a998da 100644
--- a/src/backend/cdb/cdbmutate.c
+++ b/src/backend/cdb/cdbmutate.c
@@ -312,7 +312,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query)
targetPolicy = palloc0(sizeof(GpPolicy)- sizeof(targetPolicy->attrs) + maxattrs * sizeof(targetPolicy->attrs[0]));
targetPolicy->nattrs = 0;
- targetPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, GetRandomDistPartitionNum());
+ targetPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, GetDefaultPartitionNum());
targetPolicy->ptype = POLICYTYPE_PARTITIONED;
/* Find out what the flow is partitioned on */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index bcd5384..2f4cf39 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,6 +91,11 @@
#include "postmaster/autovacuum.h"
#include "cdb/dispatcher.h"
+/*
+ * in dbsize.c
+ */
+extern int64 calculate_relation_size(Relation rel);
+
/* DestReceiver for COPY (SELECT) TO */
typedef struct
{
@@ -137,6 +142,7 @@ static void copy_in_error_callback(void *arg);
static void CopyInitPartitioningState(EState *estate);
static void CopyInitDataParser(CopyState cstate);
static bool CopyCheckIsLastLine(CopyState cstate);
+static int calculate_virtual_segment_number(List* candidateRelations);
/* ==========================================================================
* The follwing macros aid in major refactoring of data processing code (in
@@ -1556,7 +1562,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
Assert(target_policy);
- target_segment_num = target_policy->bucketnum;
+ /*
+ * For hash table we use table bucket number to request vsegs
+ * For random table, we use a fixed GUC value to request vsegs.
+ */
+ if(target_policy->nattrs > 0){
+ target_segment_num = target_policy->bucketnum;
+ }
+ else{
+ target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
+ }
pfree(target_policy);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1749,6 +1764,63 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
}
/*
+ * calculate virtual segment number for copy statement.
+ * if there is hash distributed relations exist, use the max bucket number.
+ * if all relation are random, use the data size to determine vseg number.
+ */
+static int calculate_virtual_segment_number(List* candidateOids) {
+ ListCell* le1;
+ int vsegNumber = 1;
+ int64 totalDataSize = 0;
+ bool isHashRelationExist = false;
+ int maxHashBucketNumber = 0;
+
+ foreach (le1, candidateOids)
+ {
+ Oid candidateOid = InvalidOid;
+ candidateOid = lfirst_oid(le1);
+
+ //Relation rel = (Relation)lfirst(le1);
+ Relation rel = relation_open(candidateOid, AccessShareLock);
+ if (candidateOid > 0 ) {
+ GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
+ candidateOid);
+ if(targetPolicy == NULL){
+ return GetAnalyzeVSegNumLimit();
+ }
+ if (targetPolicy->nattrs > 0) {
+ isHashRelationExist = true;
+ if(maxHashBucketNumber < targetPolicy->bucketnum){
+ maxHashBucketNumber = targetPolicy->bucketnum;
+ }
+ }
+ /*
+ * if no hash relation, we calculate the data size of all the relations.
+ */
+ if (!isHashRelationExist) {
+ totalDataSize += calculate_relation_size(rel);
+ }
+ }
+ relation_close(rel, AccessShareLock);
+ }
+
+ if (isHashRelationExist) {
+ vsegNumber = maxHashBucketNumber;
+ } else {
+ /*we allocate one virtual segment for each 128M data */
+ totalDataSize >>= 27;
+ vsegNumber = totalDataSize + 1;
+ }
+ Assert(vsegNumber > 0);
+ /*vsegNumber should be less than GetUtilPartitionNum*/
+ if(vsegNumber > GetQueryVsegNum()){
+ vsegNumber = GetQueryVsegNum();
+ }
+
+ return vsegNumber;
+}
+
+/*
* This intermediate routine exists mainly to localize the effects of setjmp
* so we don't need to plaster a lot of variables with "volatile".
*/
@@ -1809,13 +1881,21 @@ DoCopyTo(CopyState cstate)
*/
if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
{
- GpPolicy *target_policy = NULL;
int target_segment_num = 0;
+ /*
+ * copy hash table use table bucket number
+ * copy random table use table size.
+ */
+ PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
+ 0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
+ Assert(pn);
+ List *lFullRelOids = NIL;
+ lFullRelOids = all_leaf_partition_relids(pn);
+ lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id); /* root partition */
+ lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
- target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
- Assert(target_policy);
- target_segment_num = target_policy->bucketnum;
- pfree(target_policy);
+ target_segment_num = calculate_virtual_segment_number(lFullRelOids);
+ elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
CopyToDispatch(cstate);
@@ -4200,7 +4280,7 @@ CopyFrom(CopyState cstate)
if (cstate->oids && file_has_oids)
MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
}
- else if ((relstorage == RELSTORAGE_PARQUET))
+ else if (relstorage == RELSTORAGE_PARQUET)
{
tuple = NULL;
}
@@ -4231,7 +4311,7 @@ CopyFrom(CopyState cstate)
{
HeapTuple newtuple;
- if((relstorage == RELSTORAGE_PARQUET))
+ if(relstorage == RELSTORAGE_PARQUET)
{
Assert(!tuple);
elog(ERROR, "triggers are not supported on tables that use column-oriented storage");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/commands/prepare.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 76c04e9..013af9a 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -242,7 +242,7 @@ ExecuteQuery(ExecuteStmt *stmt, const char *queryString,
+ 255 * sizeof(pstmt->intoPolicy->attrs[0]));
pstmt->intoPolicy->nattrs = 0;
pstmt->intoPolicy->ptype = POLICYTYPE_PARTITIONED;
- pstmt->intoPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(stmt->into->rel, GetRandomDistPartitionNum());
+ pstmt->intoPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(stmt->into->rel, GetDefaultPartitionNum());
MemoryContextSwitchTo(oldContext);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/optimizer/util/relnode.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index 9e94901..8fb676c 100644
--- a/src/backend/optimizer/util/relnode.c
+++ b/src/backend/optimizer/util/relnode.c
@@ -142,7 +142,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind)
{
rel->cdbpolicy = (GpPolicy *) palloc(sizeof(GpPolicy));
rel->cdbpolicy->ptype = POLICYTYPE_PARTITIONED;
- rel->cdbpolicy->bucketnum = GetRandomDistPartitionNum();
+ rel->cdbpolicy->bucketnum = GetDefaultPartitionNum();
rel->cdbpolicy->nattrs = 0;
rel->cdbpolicy->attrs[0] = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/parser/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c
index ad35484..f17c90a 100644
--- a/src/backend/parser/analyze.c
+++ b/src/backend/parser/analyze.c
@@ -2724,7 +2724,7 @@ transformETDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
sizeof(p->attrs[0]));
p->ptype = POLICYTYPE_PARTITIONED;
p->nattrs = 0;
- p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetRandomDistPartitionNum());
+ p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
p->attrs[0] = 1;
}
@@ -2981,7 +2981,7 @@ transformDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
if (policy->bucketnum == -1)
{
- policy->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetRandomDistPartitionNum());
+ policy->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
}
*policyp = policy;
@@ -12187,7 +12187,7 @@ setQryDistributionPolicy(SelectStmt *stmt, Query *qry)
policy->nattrs = 0;
policy->attrs[0] = 1;
if(stmt->intoClause != NULL)
- policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetRandomDistPartitionNum());
+ policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetDefaultPartitionNum());
if (stmt->distributedBy)
{
@@ -12241,7 +12241,7 @@ setQryDistributionPolicy(SelectStmt *stmt, Query *qry)
}
else
{
- policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetRandomDistPartitionNum());
+ policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetDefaultPartitionNum());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/postmaster/identity.c
----------------------------------------------------------------------
diff --git a/src/backend/postmaster/identity.c b/src/backend/postmaster/identity.c
index b209ec5..36ea0cb 100644
--- a/src/backend/postmaster/identity.c
+++ b/src/backend/postmaster/identity.c
@@ -537,7 +537,7 @@ GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val)
}
int
-GetRandomDistPartitionNum(void)
+GetDefaultPartitionNum(void)
{
return default_hash_table_bucket_number;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index a6a8bee..56909c9 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4516,7 +4516,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&default_hash_table_bucket_number,
- 6, 1, 10000, NULL, NULL
+ 6, 1, 65535, NULL, NULL
},
{
@@ -4525,7 +4525,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_copy_from_perquery,
- 6, 1, 10000, NULL, NULL
+ 6, 1, 65535, NULL, NULL
},
{
@@ -4534,7 +4534,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_analyze_perquery_perseg_limit,
- 4, 1, 10000, NULL, NULL
+ 4, 1, 65535, NULL, NULL
},
{
@@ -4543,7 +4543,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&hawq_rm_nvseg_for_analyze_perquery_limit,
- 256, 1, 10000, NULL, NULL
+ 256, 1, 65535, NULL, NULL
},
{
@@ -6388,7 +6388,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_limit,
- 512, 1, 10000, NULL, NULL
+ 512, 1, 65535, NULL, NULL
},
{
@@ -6398,7 +6398,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_perseg_limit,
- 6, 1, 10000, NULL, NULL
+ 6, 1, 65535, NULL, NULL
},
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/include/postmaster/identity.h
----------------------------------------------------------------------
diff --git a/src/include/postmaster/identity.h b/src/include/postmaster/identity.h
index 814935e..f43ab62 100644
--- a/src/include/postmaster/identity.h
+++ b/src/include/postmaster/identity.h
@@ -91,7 +91,7 @@ extern int GetCopyFromVSegNum(void);
extern int GetRelOpt_bucket_num_fromOptions(List *options, int default_val);
extern int GetRelOpt_bucket_num_fromRel(Relation relation, int default_val);
extern int GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val);
-extern int GetRandomDistPartitionNum(void);
+extern int GetDefaultPartitionNum(void);
extern int GetHashDistPartitionNum(void);
extern int GetExternalTablePartitionNum(void);
extern int GetAllWorkerHostNum(void);
[27/50] incubator-hawq git commit: Revert "HAWQ-531. Change GUC
default value."
Posted by sh...@apache.org.
Revert "HAWQ-531. Change GUC default value."
This reverts commit 4d1a85405b87145fb931a16be9e22bd40f16eca8.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/94b7a822
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/94b7a822
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/94b7a822
Branch: refs/heads/HAWQ-459
Commit: 94b7a8225d23729d01cfc481375146d06078b601
Parents: 1897ba1
Author: hubertzhang <hz...@pivotal.io>
Authored: Thu Mar 17 15:38:46 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Thu Mar 17 15:38:46 2016 +0800
----------------------------------------------------------------------
src/backend/utils/misc/guc.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/94b7a822/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ba8e747..fa58f86 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6379,7 +6379,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_nvseg_perquery_limit,
- 512, 1, 65535, NULL, NULL
+ 1000, 1, 65535, NULL, NULL
},
{
@@ -6535,7 +6535,7 @@ static struct config_int ConfigureNamesInt[] =
NULL
},
&rm_rejectrequest_nseg_limit,
- 4, 0, 65535, NULL, NULL
+ 2, 0, 65535, NULL, NULL
},
{
[11/50] incubator-hawq git commit: HAWQ-535. hawqextract column
context does not exist error
Posted by sh...@apache.org.
HAWQ-535. hawqextract column context does not exist error
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5152fb0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5152fb0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5152fb0b
Branch: refs/heads/HAWQ-459
Commit: 5152fb0b02b6bf469d48c039d17a4780075ad578
Parents: f34f65d
Author: dlynch <dl...@gopivotal.com>
Authored: Fri Jan 29 18:13:14 2016 -0600
Committer: rlei <rl...@pivotal.io>
Committed: Tue Mar 15 10:46:31 2016 +0800
----------------------------------------------------------------------
tools/bin/hawqextract | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5152fb0b/tools/bin/hawqextract
----------------------------------------------------------------------
diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract
index be08117..dea12fb 100755
--- a/tools/bin/hawqextract
+++ b/tools/bin/hawqextract
@@ -152,12 +152,12 @@ class GpMetadataAccessor:
Example:
>>> accessor.get_aoseg_files(35709)
- >>> [{'content':'0', 'fileno':'1', 'filesize':'320'},
- ... {'content':'1', 'fileno':'1', 'filesize':'880'},
- ... {'content':'1', 'fileno':'2', 'filesize':'160'}]
+ >>> [{'fileno':'1', 'filesize':'320'},
+ ... {'fileno':'2', 'filesize':'880'},
+ ... {'fileno':'3', 'filesize':'160'}]
'''
qry = """
- SELECT content, segno as fileno, eof as filesize
+ SELECT segno as fileno, eof as filesize
FROM pg_aoseg.pg_aoseg_%d
ORDER by fileno;
""" % oid
@@ -170,14 +170,14 @@ class GpMetadataAccessor:
Example:
>>> accessor.get_paqseg_files(35709)
- >>> [{'content':'0', 'fileno':'1', 'filesize':'320'},
- ... {'content':'1', 'fileno':'1', 'filesize':'880'},
- ... {'content':'1', 'fileno':'2', 'filesize':'160'}]
+ >>> [{'fileno':'1', 'filesize':'320'},
+ ... {'fileno':'2', 'filesize':'880'},
+ ... {'fileno':'3', 'filesize':'160'}]
'''
qry = """
- SELECT content, segno as fileno, eof as filesize
+ SELECT segno as fileno, eof as filesize
FROM pg_aoseg.pg_paqseg_%d
- WHERE content >= 0 ORDER by content, fileno;
+ ORDER by fileno;
""" % oid
return self.exec_query(qry)
@@ -306,17 +306,17 @@ def extract_metadata(conn, tbname):
Example:
>>> segment_localtions
- >>> ['hdfs://127.0.0.1:9000/gpseg0', 'hdfs://127.0.0.1:9000/gpseg1']
+ >>> ['hdfs://127.0.0.1:9000/hawq_default', 'hdfs://127.0.0.1:9000/hawq_default']
>>> tablespace_oid, database_oid, relfilenode, oid
>>> (16385, 35469, 35470, 35488)
>>> accessor.get_aoseg_files(35488)
- >>> [{'content': '0', 'fileno': '1', 'filesize': '180'},
- ... {'content': '0', 'fileno': '2', 'filesize': '150'},
- ... {'content': '1', 'fileno': '1', 'filesize': '320'}]
+ >>> [{'fileno': '1', 'filesize': '180'},
+ ... {'fileno': '2', 'filesize': '150'},
+ ... {'fileno': '3', 'filesize': '320'}]
>>> get_ao_table_files(35488, 35470)
- >>> [{'path' :'/gpseg0/16385/35469/35470.1', 'size': 180},
- ... {'path' :'/gpseg0/16385/35469/35470.2', 'size': 150},
- ... {'path' :'/gpseg1/16385/35469/35470.1', 'size': 320}]
+ >>> [{'path' :'/hawq_default/16385/35469/1', 'size': 180},
+ ... {'path' :'/hawq_default/16385/35469/2', 'size': 150},
+ ... {'path' :'/hawq_default/16385/35469/3', 'size': 320}]
'''
files = []
for f in accessor.get_aoseg_files(oid):
[03/50] incubator-hawq git commit: Support --enable-orca [#105338218]
Posted by sh...@apache.org.
Support --enable-orca [#105338218]
fix gpopt interface change
fix boost_thread to use boost_thread_mt on OSX
fix version detection using ivy.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a48600a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a48600a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a48600a1
Branch: refs/heads/HAWQ-459
Commit: a48600a125eed2a3da00133d6f5f8dad22d4a6a4
Parents: 29a9beb
Author: Jacob Frank <jf...@pivotal.io>
Authored: Tue Mar 8 17:49:07 2016 -0800
Committer: C.J. Jameson <cj...@pivotal.io>
Committed: Mon Mar 14 12:24:56 2016 -0700
----------------------------------------------------------------------
configure | 34 --------------------------------
configure.in | 35 ---------------------------------
src/backend/Makefile | 19 +++++++++---------
src/backend/gpopt/Makefile | 25 ++++++++++-------------
src/backend/gpopt/gpopt.mk | 24 ++++++++++++----------
src/bin/gpcheckhdfs/Makefile | 2 +-
src/bin/gpfilesystem/hdfs/Makefile | 2 +-
7 files changed, 36 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5135536..5669406 100755
--- a/configure
+++ b/configure
@@ -731,11 +731,6 @@ with_perl
with_tcl
enable_thread_safety
INCLUDES
-ORCA_DEPENDS_OPTIMIZER_VER
-ORCA_DEPENDS_LIBGPOS_VER
-ORCA_DEPENDS_XERCES_VER
-ORCA_DEPENDS_OBJDIR_DEFAULT
-ORCA_DEPENDS_DIR_INTER
enable_orca
enable_email
enable_gphdfs
@@ -5522,35 +5517,6 @@ fi
-if test "$enable_orca" = yes ; then
-
- if test -z "$ORCA_DEPENDS_DIR" ; then
- as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
- fi
-
- if test -z "$ORCA_OBJDIR_DEFAULT" ; then
- as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
- fi
-
- if test -z "$XERCES_VER" ; then
- as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
- fi
-
- if test -z "$LIBGPOS_VER" ; then
- as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
- fi
-
- if test -z "$OPTIMIZER_VER" ; then
- as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
- fi
-
- ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
- ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
- ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
- ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
- ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
-fi
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index cda7bad..489187e 100644
--- a/configure.in
+++ b/configure.in
@@ -639,41 +639,6 @@ PGAC_ARG_BOOL(enable, orca, no, [ --enable-orca enable Pivotal Query Opt
[Define to 1 to build with Pivotal Query Optimizer. (--enable-orca)])])
AC_SUBST(enable_orca)
-if test "$enable_orca" = yes ; then
-
- if test -z "$ORCA_DEPENDS_DIR" ; then
- AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
- fi
-
- if test -z "$ORCA_OBJDIR_DEFAULT" ; then
- AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
- fi
-
- if test -z "$XERCES_VER" ; then
- AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
- fi
-
- if test -z "$LIBGPOS_VER" ; then
- AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
- fi
-
- if test -z "$OPTIMIZER_VER" ; then
- AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
- fi
-
- ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
- ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
- ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
- ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
- ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
-fi
-
-AC_SUBST(ORCA_DEPENDS_DIR_INTER)
-AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
-AC_SUBST(ORCA_DEPENDS_XERCES_VER)
-AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
-AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
-
#
# Include directories
#
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 8460b76..f462510 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,15 +52,15 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
# The backend doesn't need everything that's in LIBS, however
LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
# adding orca libraries
ifeq ($(enable_orca),yes)
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpos/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libnaucrates/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpopt/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt
+LIBS := $(LIBS) -lgpos
+LIBS := $(LIBS) -lxerces-c
+LIBS := $(LIBS) -lnaucrates
+LIBS := $(LIBS) -lgpdbcost
+LIBS := $(LIBS) -lgpopt
LIBS := $(LIBS) -L$(top_builddir)/src/backend/gpopt -ldxltranslators
endif
@@ -213,9 +213,6 @@ ifeq ($(MAKE_DLL), true)
$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
endif
endif
-ifeq ($(enable_orca),yes)
- $(MAKE) -C gpopt install
-endif
$(MAKE) -C catalog install-data
$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -228,6 +225,10 @@ endif
${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
+ifeq ($(enable_orca), yes)
+ $(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
+endif
+
install-bin: postgres $(POSTGRES_IMP) installdirs
$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/gpopt/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/Makefile b/src/backend/gpopt/Makefile
index b587ffd..a7ffa87 100644
--- a/src/backend/gpopt/Makefile
+++ b/src/backend/gpopt/Makefile
@@ -39,29 +39,24 @@ else
GPOPT_UDF_FLAGS := -shared
endif
-LIBGPOS_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpos
-LIBNAUCRATES_DIR = $(ORCA_DEPENDS_DIR_INTER)/libnaucrates
-LIBGPDBCOST_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost
-LIBGPOPT_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpopt
-
include $(top_srcdir)/src/backend/common.mk
libdxltranslators.$(LDSFX): $(SUBDIROBJS) $(OBJS)
$(CXX) $(CXXFLAGS) $(LDLIBFLAGS) \
- -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
- -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
- -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
- -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
- -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt -lpthread -o $@ \
+ -lgpos \
+ -lxerces-c \
+ -lnaucrates \
+ -lgpdbcost \
+ -lgpopt -lpthread -o $@ \
$(SUBDIROBJS) $(OBJS)
libgpoptudf.$(LDSFX): $(SUBDIROBJS)
$(CXX) $(CXXFLAGS) $(GPOPT_UDF_FLAGS) \
- -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
- -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
- -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
- -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
- -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt \
+ -lxerces-c \
+ -lgpos \
+ -lnaucrates \
+ -lgpdbcost \
+ -lgpopt \
-o $@ ./utils/funcs.o
install:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 6c77d2f..0099667 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,6 +25,8 @@
UNAME = $(shell uname)
UNAME_P = $(shell uname -p)
+UNAME_M = $(shell uname -m)
+
ARCH_OS = GPOS_$(UNAME)
ARCH_CPU = GPOS_$(UNAME_P)
@@ -41,8 +43,9 @@ ifeq "$(BLD_TYPE)" "opt"
GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
endif
-ARCH_BIT = GPOS_64BIT
-ifeq (Darwin, $(UNAME))
+ifeq (x86_64, $(UNAME_M))
+ ARCH_BIT = GPOS_64BIT
+else
ARCH_BIT = GPOS_32BIT
endif
@@ -52,14 +55,15 @@ else
ARCH_FLAGS = -m64
endif
+GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
+
+XERCES_VER = $(shell grep "\"xerces-c\"" $(GREP_SED_VAR))
+LIBGPOS_VER = $(shell grep "\"libgpos\"" $(GREP_SED_VAR))
+OPTIMIZER_VER = $(shell grep "\"optimizer\"" $(GREP_SED_VAR))
+
BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
override CPPFLAGS := -fPIC $(CPPFLAGS)
override CPPFLAGS := $(BLD_FLAGS) $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libnaucrates/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/include $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index cd81902..0b88600 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
OBJS=gpcheckhdfs.o
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index b0fb359..2399789 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS = gpfshdfs.o
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3
ifdef USE_PGXS
PGXS := $(shell pg_config --pgxs)
[39/50] incubator-hawq git commit: HAWQ-558. The query running on
random table does not push resource manager to request resource from YARN as
maimum possible resource consumption
Posted by sh...@apache.org.
HAWQ-558. The query running on random table does not push resource manager to request resource from YARN as maimum possible resource consumption
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b3a35218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b3a35218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b3a35218
Branch: refs/heads/HAWQ-459
Commit: b3a352186cbe4ebadc5b76121cbefde474fc3a53
Parents: e82ac3f
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 14:13:34 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 14:13:34 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resourcemanager.c | 83 +++++++++++++---------
1 file changed, 51 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b3a35218/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 5a01316..c585d40 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -78,6 +78,7 @@ void completeAllocRequestToBroker(int32_t *reqmem,
int32_t *reqcore,
List **preferred);
void processResourceBrokerTasks(void);
+void generateResourceRequestToResourceBroker(void);
void cleanupAllGRMContainers(void);
bool cleanedAllGRMContainers(void);
@@ -610,12 +611,15 @@ int MainHandlerLoop(void)
/* STEP 5. Handle all submitted requests through socket clients. */
processSubmittedRequests();
- /* STEP 6. Check timeout resource allocation and timeout queuing requests. */
+ /* STEP 6. Generate possible resource request to resource broker. */
+ generateResourceRequestToResourceBroker();
+
+ /* STEP 7. Check timeout resource allocation and timeout queuing requests. */
timeoutDeadResourceAllocation();
timeoutQueuedRequest();
/*
- * STEP 7. Check the status of all segment nodes, mark down if hasn't got
+ * STEP 8. Check the status of all segment nodes, mark down if hasn't got
* IMAlive message for a pre-defined period.
*/
uint64_t curtime = gettime_microsec();
@@ -628,12 +632,13 @@ int MainHandlerLoop(void)
PRESPOOL->LastCheckTime = curtime;
}
- /* STEP 8. Move all accepted GRM containers into resource pool. */
+
+ /* STEP 9. Move all accepted GRM containers into resource pool. */
moveAllAcceptedGRMContainersToResPool();
/*
- * Check if should pause dispatching resource to queries to collect
- * resource back in order to return GRM containers.
+ * STEP 10. Check if should pause dispatching resource to queries to
+ * collect resource back in order to return GRM containers.
*/
if ( DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
PRESPOOL->AddPendingContainerCount == 0 &&
@@ -654,7 +659,8 @@ int MainHandlerLoop(void)
setForcedReturnGRMContainerCount();
}
- /* STEP 9. Dispatch resource to queries and send the messages out.*/
+ /* STEP 11. Dispatch resource to queries and send the messages out.*/
+
if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 &&
PQUEMGR->toRunQueryDispatch &&
PQUEMGR->ForcedReturnGRMContainerCount == 0 &&
@@ -675,23 +681,23 @@ int MainHandlerLoop(void)
PQUEMGR->ForcedReturnGRMContainerCount);
}
- /* STEP 10. Generate output content to client connections. */
+ /* STEP 12. Generate output content to client connections. */
sendResponseToClients();
/*
- * STEP 11. Return containers to global resource manager if there some
+ * STEP 13. Return containers to global resource manager if there some
* some idle resource.
*/
timeoutIdleGRMResourceToRB();
- /* STEP 12. Notify segments to increase resource quota. */
+ /* STEP 14. Notify segments to increase resource quota. */
notifyToBeAcceptedGRMContainersToRMSEG();
- /* STEP 13. Notify segments to decrease resource. */
+ /* STEP 15. Notify segments to decrease resource. */
notifyToBeKickedGRMContainersToRMSEG();
/*
- * STEP 14. Check slaves file if the content is not checked or is
+ * STEP 16. Check slaves file if the content is not checked or is
* updated.
*/
checkSlavesFile();
@@ -2881,26 +2887,7 @@ void processResourceBrokerTasks(void)
}
}
- /* STEP 3. Generate GRM container allocation request. */
- curtime = gettime_microsec();
-
- if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 )
- {
- refreshMemoryCoreRatioLevelUsage(curtime);
- if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
- {
- PRESPOOL->LastResAcqTime = curtime;
- res = generateAllocRequestToBroker();
- if ( res != FUNC_RETURN_OK )
- {
- elog(WARNING, "Resource manager fails to allocate container "
- "from global resource manager.");
- goto exit;
- }
- }
- }
-
- /* STEP 4. Return kicked GRM containers. */
+ /* STEP 3. Return kicked GRM containers. */
curtime = gettime_microsec();
if ( !PRESPOOL->pausePhase[QUOTA_PHASE_KICKED_TO_RETURN] )
@@ -2918,7 +2905,7 @@ void processResourceBrokerTasks(void)
elog(LOG, "Paused returning GRM containers kicked to GRM.");
}
/*
- * STEP 5. Handle resource broker input as new allocated resource or
+ * STEP 4. Handle resource broker input as new allocated resource or
* cluster report, container report etc. The allocated resource
* from resource broker will be added to the resource queue
* manager and resource pool.
@@ -2940,6 +2927,38 @@ exit:
RB_handleError(res);
}
+void generateResourceRequestToResourceBroker(void)
+{
+ uint64_t curtime = 0;
+ int res = FUNC_RETURN_OK;
+
+ if ( !isCleanGRMResourceStatus() )
+ {
+
+ curtime = gettime_microsec();
+
+ if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 )
+ {
+ refreshMemoryCoreRatioLevelUsage(curtime);
+ if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
+ {
+ PRESPOOL->LastResAcqTime = curtime;
+ res = generateAllocRequestToBroker();
+ if ( res != FUNC_RETURN_OK )
+ {
+ elog(WARNING, "Resource manager fails to allocate container "
+ "from global resource manager.");
+ goto exit;
+ }
+ }
+ }
+ }
+
+exit:
+ RB_handleError(res);
+
+}
+
void cleanupAllGRMContainers(void)
{
dropAllResPoolGRMContainersToToBeKicked();
[13/50] incubator-hawq git commit: HAWQ-473. fix coverity errors
introduced by new codes
Posted by sh...@apache.org.
HAWQ-473. fix coverity errors introduced by new codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c72d0662
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c72d0662
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c72d0662
Branch: refs/heads/HAWQ-459
Commit: c72d06629f28b00b430db0bfcc79422cfd25f814
Parents: 99a307b
Author: Wen Lin <wl...@pivotal.io>
Authored: Tue Mar 15 14:55:37 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Mar 15 14:55:37 2016 +0800
----------------------------------------------------------------------
.../communication/rmcomm_RM2RMSEG.c | 16 +++++---------
src/backend/resourcemanager/requesthandler.c | 8 +++----
.../resourcebroker/resourcebroker_LIBYARN.c | 8 +++----
src/backend/resourcemanager/resourcemanager.c | 12 +++++-----
src/backend/resourcemanager/resourcepool.c | 23 +++++++-------------
5 files changed, 25 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index ca332c9..7baab7d 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -243,11 +243,9 @@ void receivedRUAliveResponse(AsyncCommMessageHandlerContext context,
add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
GET_SEGRESOURCE_HOSTNAME(segres),
description->Str);
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
/* Set the host down. */
elog(WARNING, "Resource manager sets host %s from up to down "
@@ -304,11 +302,9 @@ void sentRUAliveError(AsyncCommMessageHandlerContext context)
add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
GET_SEGRESOURCE_HOSTNAME(segres),
description->Str);
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
/* Set the host down. */
elog(LOG, "Resource manager sets host %s from up to down "
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index a06d169..694c9cd 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -1014,11 +1014,9 @@ bool handleRMRequestSegmentIsDown(void **arg)
add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
hostname,
description->Str);
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
/* Set the host down. */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index d446a6d..cdbe0d5 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -732,11 +732,9 @@ int handleRB2RM_ClusterReport(void)
GET_SEGRESOURCE_HOSTNAME(segres),
SEGMENT_STATUS_DOWN,
(description->Len > 0)?description->Str:"");
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
}
freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 138c5a0..cb1eecc 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -2557,9 +2557,9 @@ void updateStatusOfAllNodes()
for(uint32_t idx = 0; idx < PRESPOOL->SegmentIDCounter; idx++)
{
node = getSegResource(idx);
+ Assert(node != NULL);
uint8_t oldStatus = node->Stat->FTSAvailable;
- if (node != NULL &&
- (curtime - node->LastUpdateTime >
+ if ( (curtime - node->LastUpdateTime >
1000000LL * rm_segment_heartbeat_timeout) &&
(node->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0)
{
@@ -2591,11 +2591,9 @@ void updateStatusOfAllNodes()
add_segment_history_row(idx + REGISTRATION_ORDER_OFFSET,
GET_SEGRESOURCE_HOSTNAME(node),
(description->Len > 0)?description->Str:"");
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
elog(WARNING, "Resource manager sets host %s heartbeat timeout.",
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index d3a01a4..ff5f34f 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -992,11 +992,8 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
SEG_STATUS_DESCRIPTION_UP:description->Str);
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
if (segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
@@ -1243,11 +1240,9 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
GET_SEGRESOURCE_HOSTNAME(segresource),
IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
SEG_STATUS_DESCRIPTION_UP:description->Str);
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
}
@@ -1538,11 +1533,9 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
(description->Len > 0)?description->Str:"");
- if (description != NULL)
- {
- freeSimpleStringContent(description);
- rm_pfree(PCONTEXT, description);
- }
+
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
int32_t curratio = 0;
[43/50] incubator-hawq git commit: HAWQ-502 Enable ORCA [#105338218]
Posted by sh...@apache.org.
HAWQ-502 Enable ORCA [#105338218]
This adds `--enable-orca` in autoconf.
This commit is the first step in fixing apache/incuabor-hawq#420 .
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c67a1a43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c67a1a43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c67a1a43
Branch: refs/heads/HAWQ-459
Commit: c67a1a43c7159eda9d5d6f70110d6db9425110f0
Parents: 1d9eb5e
Author: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Authored: Fri Mar 18 17:01:37 2016 -0700
Committer: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Committed: Fri Mar 18 17:01:37 2016 -0700
----------------------------------------------------------------------
configure | 27 ---------------------------
configure.in | 27 ---------------------------
src/backend/Makefile | 6 +++---
src/backend/gpopt/gpopt.mk | 18 +++++++++++++-----
4 files changed, 16 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5135536..9dffef3 100755
--- a/configure
+++ b/configure
@@ -731,9 +731,6 @@ with_perl
with_tcl
enable_thread_safety
INCLUDES
-ORCA_DEPENDS_OPTIMIZER_VER
-ORCA_DEPENDS_LIBGPOS_VER
-ORCA_DEPENDS_XERCES_VER
ORCA_DEPENDS_OBJDIR_DEFAULT
ORCA_DEPENDS_DIR_INTER
enable_orca
@@ -5523,32 +5520,8 @@ fi
if test "$enable_orca" = yes ; then
-
- if test -z "$ORCA_DEPENDS_DIR" ; then
- as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
- fi
-
- if test -z "$ORCA_OBJDIR_DEFAULT" ; then
- as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
- fi
-
- if test -z "$XERCES_VER" ; then
- as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
- fi
-
- if test -z "$LIBGPOS_VER" ; then
- as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
- fi
-
- if test -z "$OPTIMIZER_VER" ; then
- as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
- fi
-
ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
- ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
- ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
- ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
fi
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index cda7bad..13260ef 100644
--- a/configure.in
+++ b/configure.in
@@ -640,39 +640,12 @@ PGAC_ARG_BOOL(enable, orca, no, [ --enable-orca enable Pivotal Query Opt
AC_SUBST(enable_orca)
if test "$enable_orca" = yes ; then
-
- if test -z "$ORCA_DEPENDS_DIR" ; then
- AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
- fi
-
- if test -z "$ORCA_OBJDIR_DEFAULT" ; then
- AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
- fi
-
- if test -z "$XERCES_VER" ; then
- AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
- fi
-
- if test -z "$LIBGPOS_VER" ; then
- AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
- fi
-
- if test -z "$OPTIMIZER_VER" ; then
- AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
- fi
-
ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
- ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
- ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
- ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
fi
AC_SUBST(ORCA_DEPENDS_DIR_INTER)
AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
-AC_SUBST(ORCA_DEPENDS_XERCES_VER)
-AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
-AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
#
# Include directories
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 8460b76..18ed91a 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -213,9 +213,6 @@ ifeq ($(MAKE_DLL), true)
$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
endif
endif
-ifeq ($(enable_orca),yes)
- $(MAKE) -C gpopt install
-endif
$(MAKE) -C catalog install-data
$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -228,6 +225,9 @@ endif
${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
+ifeq ($(enable_orca), yes)
+ $(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
+endif
install-bin: postgres $(POSTGRES_IMP) installdirs
$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 6c77d2f..1fa7d81 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,6 +25,7 @@
UNAME = $(shell uname)
UNAME_P = $(shell uname -p)
+UNAME_M = $(shell uname -m)
ARCH_OS = GPOS_$(UNAME)
ARCH_CPU = GPOS_$(UNAME_P)
@@ -41,8 +42,9 @@ ifeq "$(BLD_TYPE)" "opt"
GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
endif
-ARCH_BIT = GPOS_64BIT
-ifeq (Darwin, $(UNAME))
+ifeq (x86_64, $(UNAME_M))
+ ARCH_BIT = GPOS_64BIT
+else
ARCH_BIT = GPOS_32BIT
endif
@@ -52,12 +54,18 @@ else
ARCH_FLAGS = -m64
endif
+GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
+
+XERCES_VER = $(shell grep "\"xerces-c\"" $(GREP_SED_VAR))
+LIBGPOS_VER = $(shell grep "\"libgpos\"" $(GREP_SED_VAR))
+OPTIMIZER_VER = $(shell grep "\"optimizer\"" $(GREP_SED_VAR))
+
BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
override CPPFLAGS := -fPIC $(CPPFLAGS)
override CPPFLAGS := $(BLD_FLAGS) $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)
override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)
[37/50] incubator-hawq git commit: HAWQ-556. Resource manager uses
unadjusted water level guc for allocating GRM resource containers
Posted by sh...@apache.org.
HAWQ-556. Resource manager uses unadjusted water level guc for allocating GRM resource containers
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f38ad114
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f38ad114
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f38ad114
Branch: refs/heads/HAWQ-459
Commit: f38ad11472cd3c8084ac156123583239e192be5b
Parents: 17af44c
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:49:09 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:49:09 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resourcemanager.c | 2 +-
src/backend/resourcemanager/resqueuemanager.c | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f38ad114/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 21bf0d1..5a01316 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -2065,7 +2065,7 @@ int generateAllocRequestToBroker(void)
}
/* Decide water level of resource. */
- int wlevel = hasWorkload ? rm_min_resource_perseg : 0;
+ int wlevel = hasWorkload ? PQUEMGR->ActualMinGRMContainerPerSeg : 0;
switch( DRMGlobalInstance->ImpType )
{
case YARN_LIBYARN:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f38ad114/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index cbf824e..538c2ad 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -5213,8 +5213,10 @@ void getIdleResourceRequest(int32_t *mem, double *core)
Assert(PRESPOOL->ClusterMemoryCoreRatio > 0);
*mem = PRESPOOL->ClusterMemoryCoreRatio *
PRESPOOL->AvailNodeCount *
- rm_min_resource_perseg;
- *core = 1.0 * PRESPOOL->AvailNodeCount * rm_min_resource_perseg;
+ PQUEMGR->ActualMinGRMContainerPerSeg;
+ *core = 1.0 *
+ PRESPOOL->AvailNodeCount *
+ PQUEMGR->ActualMinGRMContainerPerSeg;
}
void setForcedReturnGRMContainerCount(void)
[15/50] incubator-hawq git commit: HAWQ-542. Passing empty string to
attribute vseg_resource_quota makes default value set instead of reporting
error
Posted by sh...@apache.org.
HAWQ-542. Passing empty string to attribute vseg_resource_quota makes default value set instead of reporting error
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/31eb645b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/31eb645b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/31eb645b
Branch: refs/heads/HAWQ-459
Commit: 31eb645b44bbd8ec453dbcb33a491c16f3f69178
Parents: ca2a770
Author: YI JIN <yj...@pivotal.io>
Authored: Wed Mar 16 13:37:55 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Wed Mar 16 13:37:55 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resqueuemanager.c | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31eb645b/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 7bc0e7b..cbf824e 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -346,6 +346,17 @@ int shallowparseResourceQueueWithAttributes(List *rawattr,
case RSQ_DDL_ATTR_NVSEG_UPPER_LIMIT_PERSEG:
case RSQ_DDL_ATTR_NVSEG_LOWER_LIMIT_PERSEG:
{
+
+ /* The empty value is not allowed. */
+ if ( SimpleStringEmpty(&(property->Val)) )
+ {
+ snprintf(errorbuf, errorbufsize,
+ "the value of %s cannot be empty",
+ RSQDDLAttrNames[attrindex]);
+ elog(WARNING, "Resource manager failed parsing attribute, %s",
+ errorbuf);
+ return RMDDL_WRONG_ATTRVALUE;
+ }
/*
* Build property.
*
[10/50] incubator-hawq git commit: HAWQ-534. A query resource quota
can be calculated to from 0 to 0
Posted by sh...@apache.org.
HAWQ-534. A query resource quota can be calculated to from 0 to 0
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f34f65de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f34f65de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f34f65de
Branch: refs/heads/HAWQ-459
Commit: f34f65de1a2db5de7d1768c25f8eb6fa15adf578
Parents: a7ad375
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Mar 15 13:36:58 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Mar 15 13:36:58 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resqueuemanager.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f34f65de/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 9af0d7c..7bc0e7b 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -3868,7 +3868,7 @@ int computeQueryQuota_EVEN(DynResourceQueueTrack track,
*segnummin = reservsegnum;
*segnummin = *segnummin > *segnum ? *segnum : *segnummin;
- Assert( *segnummin > 0 && *segnummin <= *segnum );
+ Assert( *segnummin >= 0 && *segnummin <= *segnum );
return FUNC_RETURN_OK;
}
[29/50] incubator-hawq git commit: HAWQ-550. Auto clean up records in
gp_configuration_history longer than a GUC value, default is 365 days
Posted by sh...@apache.org.
HAWQ-550. Auto clean up records in gp_configuration_history longer than a GUC value, default is 365 days
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/64663eff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/64663eff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/64663eff
Branch: refs/heads/HAWQ-459
Commit: 64663eff91a03c2c95bb1f4f145bdf502253a805
Parents: dfc6731
Author: Wen Lin <wl...@pivotal.io>
Authored: Fri Mar 18 09:37:38 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Fri Mar 18 09:37:38 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbvars.c | 1 +
.../resourcemanager/include/resourcepool.h | 3 +
.../resourcemanager/requesthandler_RMSEG.c | 5 +-
src/backend/resourcemanager/resourcemanager.c | 2 +
src/backend/resourcemanager/resourcepool.c | 74 +++++++++++++++++++-
src/backend/utils/misc/guc.c | 9 +++
src/include/cdb/cdbvars.h | 1 +
7 files changed, 92 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index fbb4bc0..b4a9f3d 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -297,6 +297,7 @@ int seg_addr_port;
char *dfs_url;
char *master_directory;
char *seg_directory;
+int segment_history_keep_period;
/* HAWQ 2.0 resource manager GUCs */
int rm_master_port;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index 5feedd3..f965c43 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -670,6 +670,9 @@ void adjustMemoryCoreValue(uint32_t *memorymb, uint32_t *core);
/* Clean up gp_segment_configuration */
void cleanup_segment_config(void);
+/* Clean up gp_configuration_history */
+void cleanup_segment_config_history(void);
+
#define SEG_STATUS_DESCRIPTION_UP "segment is up"
/* update a segment's status in gp_segment_configuration table */
void update_segment_status(int32_t id, char status, char* description);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/requesthandler_RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler_RMSEG.c b/src/backend/resourcemanager/requesthandler_RMSEG.c
index 90946a0..12bd8ac 100644
--- a/src/backend/resourcemanager/requesthandler_RMSEG.c
+++ b/src/backend/resourcemanager/requesthandler_RMSEG.c
@@ -307,7 +307,8 @@ bool handleRMSEGRequestRUAlive(void **arg)
{
if (retry == 0)
{
- elog(LOG, "Segment's postmaster is down, PQconnectdb result : %d, %s",
+ elog(LOG, "Segment receives RUAlive from master "
+ "and postmaster is down, PQconnectdb result : %d, %s",
libpqres,
PQerrorMessage(conn));
/* Don't send IMAlive anymore */
@@ -322,7 +323,7 @@ bool handleRMSEGRequestRUAlive(void **arg)
}
else
{
- elog(DEBUG3, "Segment's postmaster is healthy.");
+ elog(LOG, "Segment receives RUAlive from master and postmaster is healthy.");
break;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index cb1eecc..45a669e 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -479,6 +479,8 @@ int ResManagerMainServer2ndPhase(void)
/* Clean up gp_segment_configuration table. */
cleanup_segment_config();
+ cleanup_segment_config_history();
+
/*
* register itself into gp_segment_configuration table
* master internal id is 0, segment id starts from 1
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index ff5f34f..ca275f5 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -28,6 +28,7 @@
#include "gp-libpq-int.h"
#include "utils/builtins.h"
#include "catalog/pg_proc.h"
+#include "cdb/cdbvars.h"
void addSegResourceAvailIndex(SegResource segres);
void addSegResourceAllocIndex(SegResource segres);
@@ -471,6 +472,77 @@ cleanup:
}
/*
+ * Remove records in gp_configuration_history that are longer than
+ * a period defined in GUC, the default values is 365 days.
+ */
+void cleanup_segment_config_history(){
+ int libpqres = CONNECTION_OK;
+ PGconn *conn = NULL;
+ char conninfo[512];
+ PQExpBuffer sql = NULL;
+ PGresult* result = NULL;
+
+ sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
+ "dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
+ conn = PQconnectdb(conninfo);
+ if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
+ {
+ elog(WARNING, "Fail to connect database when cleanup "
+ "segment history catalog table, error code: %d, %s",
+ libpqres,
+ PQerrorMessage(conn));
+ PQfinish(conn);
+ return;
+ }
+
+ result = PQexec(conn, "BEGIN");
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ "BEGIN",
+ PQresultErrorMessage(result));
+ goto cleanup;
+ }
+ PQclear(result);
+
+ sql = createPQExpBuffer();
+ appendPQExpBuffer(sql,"DELETE FROM gp_configuration_history WHERE "
+ "current_timestamp - time > interval '%d days'",
+ segment_history_keep_period);
+ result = PQexec(conn, sql->data);
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ sql->data,
+ PQresultErrorMessage(result));
+ goto cleanup;
+ }
+ PQclear(result);
+
+ result = PQexec(conn, "COMMIT");
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ "COMMIT",
+ PQresultErrorMessage(result));
+ goto cleanup;
+ }
+
+ elog(LOG, "Cleanup segment configuration history catalog table successfully, "
+ "keep period: recent %d days.", segment_history_keep_period);
+
+cleanup:
+ if(sql)
+ destroyPQExpBuffer(sql);
+ if(result)
+ PQclear(result);
+ PQfinish(conn);
+}
+
+/*
* Remove all entries in gp_configuration_history.
*
* gp_remove_segment_history()
@@ -500,7 +572,7 @@ gp_remove_segment_history(PG_FUNCTION_ARGS)
if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
{
elog(WARNING, "Fail to connect database when cleanup "
- "segment configuration catalog table, error code: %d, %s",
+ "segment history catalog table, error code: %d, %s",
libpqres,
PQerrorMessage(conn));
PQfinish(conn);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fa58f86..54ef13f 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6264,6 +6264,15 @@ static struct config_int ConfigureNamesInt[] =
},
{
+ {"hawq_segment_history_keep_period", PGC_POSTMASTER, RESOURCES_MGM,
+ gettext_noop("period of storing rows in segment_configuration_history"),
+ NULL
+ },
+ &segment_history_keep_period,
+ 365, 1, INT_MAX, NULL, NULL
+ },
+
+ {
{"hawq_rm_master_domain_port", PGC_POSTMASTER, RESOURCES_MGM,
gettext_noop("resource manager master domain socket port number"),
NULL
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/include/cdb/cdbvars.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index fbe6251..3129f88 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -1148,6 +1148,7 @@ extern int seg_addr_port;
extern char *dfs_url;
extern char *master_directory;
extern char *seg_directory;
+extern int segment_history_keep_period;
/* HAWQ 2.0 resource manager GUCs */
extern int rm_master_domain_port;
[47/50] incubator-hawq git commit: Revert "Revert "HAWQ-532. Optimise
vseg number for copy to statement.""
Posted by sh...@apache.org.
Revert "Revert "HAWQ-532. Optimise vseg number for copy to statement.""
This reverts commit 86d9b0c03de349398ddb0337f7ac5dac3b58d2b9.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/695606ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/695606ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/695606ff
Branch: refs/heads/HAWQ-459
Commit: 695606ff442afcf4e9317451e02e3a6a31f900ff
Parents: 802d537
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 14:24:41 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:29:41 2016 +0800
----------------------------------------------------------------------
src/backend/commands/copy.c | 97 ++++++++++++++++++++++++++++++++++++----
1 file changed, 89 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/695606ff/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index bcd5384..e37fb83 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,6 +91,11 @@
#include "postmaster/autovacuum.h"
#include "cdb/dispatcher.h"
+/*
+ * in dbsize.c
+ */
+extern int64 calculate_relation_size(Relation rel);
+
/* DestReceiver for COPY (SELECT) TO */
typedef struct
{
@@ -137,6 +142,7 @@ static void copy_in_error_callback(void *arg);
static void CopyInitPartitioningState(EState *estate);
static void CopyInitDataParser(CopyState cstate);
static bool CopyCheckIsLastLine(CopyState cstate);
+static int calculate_virtual_segment_number(List* candidateRelations);
/* ==========================================================================
* The follwing macros aid in major refactoring of data processing code (in
@@ -1556,7 +1562,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
Assert(target_policy);
- target_segment_num = target_policy->bucketnum;
+ /*
+ * For hash table we use table bucket number to request vsegs
+ * For random table, we use a fixed GUC value to request vsegs.
+ */
+ if(target_policy->nattrs > 0){
+ target_segment_num = target_policy->bucketnum;
+ }
+ else{
+ target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
+ }
pfree(target_policy);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1749,6 +1764,63 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
}
/*
+ * calculate virtual segment number for copy statement.
+ * if there is hash distributed relations exist, use the max bucket number.
+ * if all relation are random, use the data size to determine vseg number.
+ */
+static int calculate_virtual_segment_number(List* candidateOids) {
+ ListCell* le1;
+ int vsegNumber = 1;
+ int64 totalDataSize = 0;
+ bool isHashRelationExist = false;
+ int maxHashBucketNumber = 0;
+
+ foreach (le1, candidateOids)
+ {
+ Oid candidateOid = InvalidOid;
+ candidateOid = lfirst_oid(le1);
+
+ //Relation rel = (Relation)lfirst(le1);
+ Relation rel = relation_open(candidateOid, AccessShareLock);
+ if (candidateOid > 0 ) {
+ GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
+ candidateOid);
+ if(targetPolicy == NULL){
+ return GetAnalyzeVSegNumLimit();
+ }
+ if (targetPolicy->nattrs > 0) {
+ isHashRelationExist = true;
+ if(maxHashBucketNumber < targetPolicy->bucketnum){
+ maxHashBucketNumber = targetPolicy->bucketnum;
+ }
+ }
+ /*
+ * if no hash relation, we calculate the data size of all the relations.
+ */
+ if (!isHashRelationExist) {
+ totalDataSize += calculate_relation_size(rel);
+ }
+ }
+ relation_close(rel, AccessShareLock);
+ }
+
+ if (isHashRelationExist) {
+ vsegNumber = maxHashBucketNumber;
+ } else {
+ /*we allocate one virtual segment for each 128M data */
+ totalDataSize >>= 27;
+ vsegNumber = totalDataSize + 1;
+ }
+ Assert(vsegNumber > 0);
+ /*vsegNumber should be less than GetUtilPartitionNum*/
+ if(vsegNumber > GetQueryVsegNum()){
+ vsegNumber = GetQueryVsegNum();
+ }
+
+ return vsegNumber;
+}
+
+/*
* This intermediate routine exists mainly to localize the effects of setjmp
* so we don't need to plaster a lot of variables with "volatile".
*/
@@ -1809,13 +1881,22 @@ DoCopyTo(CopyState cstate)
*/
if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
{
- GpPolicy *target_policy = NULL;
int target_segment_num = 0;
+ /*
+ * copy hash table use table bucket number
+ * copy random table use table size.
+ */
+ PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
+ 0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
+ List *lFullRelOids = NIL;
+ if(pn){
+ lFullRelOids = all_leaf_partition_relids(pn);
+ lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+ }
+ lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
- target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
- Assert(target_policy);
- target_segment_num = target_policy->bucketnum;
- pfree(target_policy);
+ target_segment_num = calculate_virtual_segment_number(lFullRelOids);
+ elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
CopyToDispatch(cstate);
@@ -4200,7 +4281,7 @@ CopyFrom(CopyState cstate)
if (cstate->oids && file_has_oids)
MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
}
- else if ((relstorage == RELSTORAGE_PARQUET))
+ else if (relstorage == RELSTORAGE_PARQUET)
{
tuple = NULL;
}
@@ -4231,7 +4312,7 @@ CopyFrom(CopyState cstate)
{
HeapTuple newtuple;
- if((relstorage == RELSTORAGE_PARQUET))
+ if(relstorage == RELSTORAGE_PARQUET)
{
Assert(!tuple);
elog(ERROR, "triggers are not supported on tables that use column-oriented storage");
[04/50] incubator-hawq git commit: Revert "Support --enable-orca
[#105338218]"
Posted by sh...@apache.org.
Revert "Support --enable-orca [#105338218]"
This reverts commit a48600a125eed2a3da00133d6f5f8dad22d4a6a4.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d5e7b927
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d5e7b927
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d5e7b927
Branch: refs/heads/HAWQ-459
Commit: d5e7b927020b80b812cd66f1416336d649f3bcc9
Parents: a48600a
Author: C.J. Jameson <cj...@pivotal.io>
Authored: Mon Mar 14 14:34:00 2016 -0700
Committer: C.J. Jameson <cj...@pivotal.io>
Committed: Mon Mar 14 14:34:00 2016 -0700
----------------------------------------------------------------------
configure | 34 ++++++++++++++++++++++++++++++++
configure.in | 35 +++++++++++++++++++++++++++++++++
src/backend/Makefile | 19 +++++++++---------
src/backend/gpopt/Makefile | 25 +++++++++++++----------
src/backend/gpopt/gpopt.mk | 24 ++++++++++------------
src/bin/gpcheckhdfs/Makefile | 2 +-
src/bin/gpfilesystem/hdfs/Makefile | 2 +-
7 files changed, 105 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5669406..5135536 100755
--- a/configure
+++ b/configure
@@ -731,6 +731,11 @@ with_perl
with_tcl
enable_thread_safety
INCLUDES
+ORCA_DEPENDS_OPTIMIZER_VER
+ORCA_DEPENDS_LIBGPOS_VER
+ORCA_DEPENDS_XERCES_VER
+ORCA_DEPENDS_OBJDIR_DEFAULT
+ORCA_DEPENDS_DIR_INTER
enable_orca
enable_email
enable_gphdfs
@@ -5517,6 +5522,35 @@ fi
+if test "$enable_orca" = yes ; then
+
+ if test -z "$ORCA_DEPENDS_DIR" ; then
+ as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
+ fi
+
+ if test -z "$ORCA_OBJDIR_DEFAULT" ; then
+ as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
+ fi
+
+ if test -z "$XERCES_VER" ; then
+ as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
+ fi
+
+ if test -z "$LIBGPOS_VER" ; then
+ as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
+ fi
+
+ if test -z "$OPTIMIZER_VER" ; then
+ as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
+ fi
+
+ ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
+ ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
+ ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
+ ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
+ ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
+fi
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index 489187e..cda7bad 100644
--- a/configure.in
+++ b/configure.in
@@ -639,6 +639,41 @@ PGAC_ARG_BOOL(enable, orca, no, [ --enable-orca enable Pivotal Query Opt
[Define to 1 to build with Pivotal Query Optimizer. (--enable-orca)])])
AC_SUBST(enable_orca)
+if test "$enable_orca" = yes ; then
+
+ if test -z "$ORCA_DEPENDS_DIR" ; then
+ AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
+ fi
+
+ if test -z "$ORCA_OBJDIR_DEFAULT" ; then
+ AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
+ fi
+
+ if test -z "$XERCES_VER" ; then
+ AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
+ fi
+
+ if test -z "$LIBGPOS_VER" ; then
+ AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
+ fi
+
+ if test -z "$OPTIMIZER_VER" ; then
+ AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
+ fi
+
+ ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
+ ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
+ ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
+ ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
+ ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
+fi
+
+AC_SUBST(ORCA_DEPENDS_DIR_INTER)
+AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
+AC_SUBST(ORCA_DEPENDS_XERCES_VER)
+AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
+AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
+
#
# Include directories
#
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index f462510..8460b76 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,15 +52,15 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
# The backend doesn't need everything that's in LIBS, however
LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
# adding orca libraries
ifeq ($(enable_orca),yes)
-LIBS := $(LIBS) -lgpos
-LIBS := $(LIBS) -lxerces-c
-LIBS := $(LIBS) -lnaucrates
-LIBS := $(LIBS) -lgpdbcost
-LIBS := $(LIBS) -lgpopt
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpos/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libnaucrates/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpopt/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt
LIBS := $(LIBS) -L$(top_builddir)/src/backend/gpopt -ldxltranslators
endif
@@ -213,6 +213,9 @@ ifeq ($(MAKE_DLL), true)
$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
endif
endif
+ifeq ($(enable_orca),yes)
+ $(MAKE) -C gpopt install
+endif
$(MAKE) -C catalog install-data
$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -225,10 +228,6 @@ endif
${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
-ifeq ($(enable_orca), yes)
- $(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
-endif
-
install-bin: postgres $(POSTGRES_IMP) installdirs
$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/gpopt/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/Makefile b/src/backend/gpopt/Makefile
index a7ffa87..b587ffd 100644
--- a/src/backend/gpopt/Makefile
+++ b/src/backend/gpopt/Makefile
@@ -39,24 +39,29 @@ else
GPOPT_UDF_FLAGS := -shared
endif
+LIBGPOS_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpos
+LIBNAUCRATES_DIR = $(ORCA_DEPENDS_DIR_INTER)/libnaucrates
+LIBGPDBCOST_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost
+LIBGPOPT_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpopt
+
include $(top_srcdir)/src/backend/common.mk
libdxltranslators.$(LDSFX): $(SUBDIROBJS) $(OBJS)
$(CXX) $(CXXFLAGS) $(LDLIBFLAGS) \
- -lgpos \
- -lxerces-c \
- -lnaucrates \
- -lgpdbcost \
- -lgpopt -lpthread -o $@ \
+ -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
+ -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
+ -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
+ -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
+ -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt -lpthread -o $@ \
$(SUBDIROBJS) $(OBJS)
libgpoptudf.$(LDSFX): $(SUBDIROBJS)
$(CXX) $(CXXFLAGS) $(GPOPT_UDF_FLAGS) \
- -lxerces-c \
- -lgpos \
- -lnaucrates \
- -lgpdbcost \
- -lgpopt \
+ -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
+ -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
+ -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
+ -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
+ -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt \
-o $@ ./utils/funcs.o
install:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 0099667..6c77d2f 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,8 +25,6 @@
UNAME = $(shell uname)
UNAME_P = $(shell uname -p)
-UNAME_M = $(shell uname -m)
-
ARCH_OS = GPOS_$(UNAME)
ARCH_CPU = GPOS_$(UNAME_P)
@@ -43,9 +41,8 @@ ifeq "$(BLD_TYPE)" "opt"
GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
endif
-ifeq (x86_64, $(UNAME_M))
- ARCH_BIT = GPOS_64BIT
-else
+ARCH_BIT = GPOS_64BIT
+ifeq (Darwin, $(UNAME))
ARCH_BIT = GPOS_32BIT
endif
@@ -55,15 +52,14 @@ else
ARCH_FLAGS = -m64
endif
-GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
-
-XERCES_VER = $(shell grep "\"xerces-c\"" $(GREP_SED_VAR))
-LIBGPOS_VER = $(shell grep "\"libgpos\"" $(GREP_SED_VAR))
-OPTIMIZER_VER = $(shell grep "\"optimizer\"" $(GREP_SED_VAR))
-
BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
override CPPFLAGS := -fPIC $(CPPFLAGS)
override CPPFLAGS := $(BLD_FLAGS) $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libnaucrates/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/include $(CPPFLAGS)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index 0b88600..cd81902 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
OBJS=gpcheckhdfs.o
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index 2399789..b0fb359 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS = gpfshdfs.o
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
ifdef USE_PGXS
PGXS := $(shell pg_config --pgxs)
[16/50] incubator-hawq git commit: HAWQ-529. Allocate resource for
udf in resource negotiator.
Posted by sh...@apache.org.
HAWQ-529. Allocate resource for udf in resource negotiator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8e38e844
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8e38e844
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8e38e844
Branch: refs/heads/HAWQ-459
Commit: 8e38e844f0d2a1c45f7d5cd25a8f5b639c8b6fb7
Parents: 31eb645
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 10:51:07 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 10:51:07 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdatalocality.c | 194 ++++++++++++++++++++----------
src/backend/executor/spi.c | 72 +++++++++--
src/backend/optimizer/plan/planner.c | 57 ++++++++-
src/backend/optimizer/util/clauses.c | 11 +-
src/backend/optimizer/util/walkers.c | 1 +
src/include/cdb/cdbdatalocality.h | 19 ++-
src/include/executor/spi.h | 7 ++
src/include/optimizer/planner.h | 4 +
8 files changed, 289 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index c1f829b..d15d6b0 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,6 +33,9 @@
#include "access/filesplit.h"
#include "access/parquetsegfiles.h"
#include "catalog/catalog.h"
+#include "catalog/catquery.h"
+#include "catalog/pg_inherits.h"
+#include "catalog/pg_proc.h"
#include "cdb/cdbdatalocality.h"
#include "cdb/cdbutil.h"
#include "cdb/cdbvars.h"
@@ -41,6 +44,7 @@
#include "utils/tqual.h"
#include "utils/memutils.h"
#include "executor/execdesc.h"
+#include "executor/spi.h"
#include "nodes/nodes.h"
#include "nodes/parsenodes.h"
#include "optimizer/walkers.h"
@@ -54,11 +58,7 @@
#include "catalog/pg_proc.h"
#include "postgres.h"
#include "resourcemanager/utils/hashtable.h"
-#include "catalog/pg_inherits.h"
-//#include "utils/misc/guc.c"
-#define PRONAME 1
-#define PROISAGG 5
/* We need to build a mapping from host name to host index */
extern bool optimizer; /* Enable the optimizer */
@@ -372,7 +372,7 @@ static Block_Host_Index * update_data_dist_stat(
static HostDataVolumeInfo *search_host_in_stat_context(
split_to_segment_mapping_context *context, char *hostname);
-static bool IsAggFunction(char* funcName);
+static bool IsBuildInFunction(Oid funcOid);
static bool allocate_hash_relation(Relation_Data* rel_data,
Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -630,39 +630,36 @@ static void collect_range_tables(Query *query, List* full_range_table,
/*
*
*/
-static bool IsAggFunction(char* funcName) {
- if (funcName == NULL) {
- return false;
- }
- Relation pg_proc_rel;
- TupleDesc pg_proc_dsc;
- HeapTuple tuple;
- SysScanDesc pg_proc_scan;
-
- pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
- pg_proc_dsc = RelationGetDescr(pg_proc_rel);
- ScanKeyData skey;
+static bool IsBuildInFunction(Oid foid) {
- ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
- F_NAMEEQ, CStringGetDatum(funcName));
-
- pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
- ActiveSnapshot, 1, &skey);
- while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+ cqContext *pcqCtx;
+ HeapTuple readtup = NULL;
+ HeapTuple procedureTuple;
+ Form_pg_proc procedureStruct;
- bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
- systable_endscan(pg_proc_scan);
- heap_close(pg_proc_rel, AccessShareLock);
- if (isAgg) {
- return true;
- } else {
- return false;
- }
+ /*
+ * get the procedure tuple corresponding to the given function Oid
+ */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_proc "
+ " WHERE oid = :1 ",
+ ObjectIdGetDatum(foid)));
+
+ procedureTuple = caql_getnext(pcqCtx);
+
+ if (!HeapTupleIsValid(procedureTuple))
+ elog(ERROR, "cache lookup failed for function %u", foid);
+ procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
+ caql_endscan(pcqCtx);
+ /* we treat proc namespace = 11 to build in function.*/
+ if (procedureStruct->pronamespace == 11) {
+ return true;
+ } else {
+ return false;
}
- systable_endscan(pg_proc_scan);
- heap_close(pg_proc_rel, AccessShareLock);
- return true;
}
+
/*
*
*/
@@ -676,25 +673,6 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
foreach(old_lc, *range_tables)
{
RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
- if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
- *isTableFunctionExists = true;
- }
- if (entry->rtekind == RTE_SUBQUERY) {
- Query* subQuery = entry->subquery;
- ListCell *lc;
- foreach(lc, subQuery->targetList)
- {
- TargetEntry *te = (TargetEntry *) lfirst(lc);
- bool isAggFunc = IsAggFunction(te->resname);
- // if target list of subquery contains non aggregate function,
- // then we consider the query contains and use default_segment_num guc
- // as the number of virtual segment
- if (!isAggFunc) {
- *isTableFunctionExists = true;
- }
- }
-
- }
if (entry->rtekind != RTE_RELATION) {
continue;
}
@@ -3958,11 +3936,54 @@ static void cleanup_allocation_algorithm(
}
/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,
+ udf_collector_context *context) {
+ if (node == NULL) {
+ return false;
+ }
+
+ if (IsA(node, Query)) {
+ return query_tree_walker((Query *) node, udf_collector_walker,
+ (void *) context,
+ QTW_EXAMINE_RTES);
+ }
+
+ /*For Aggref, we don't consider it as udf.*/
+
+ if(IsA(node,FuncExpr)){
+ if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
+ context->udf_exist = true;
+ }
+ return false;
+ }
+
+ return expression_tree_walker(node, udf_collector_walker,
+ (void *) context);
+
+ return false;
+}
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context) {
+
+ query_tree_walker(query, udf_collector_walker, (void *) context,
+ QTW_EXAMINE_RTES);
+
+ return;
+}
+
+
+/*
* calculate_planner_segment_num
+ * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
*/
SplitAllocResult *
calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
+ List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
SplitAllocResult *result = NULL;
QueryResource *resource = NULL;
QueryResourceParameters *resource_parameters = NULL;
@@ -3971,14 +3992,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
List *alloc_result = NIL;
split_to_segment_mapping_context context;
- int planner_segments = -1; /*virtual segments number for explain statement */
+ int planner_segments = 0; /*virtual segments number for explain statement */
result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
result->resource = NULL;
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = -1;
+ result->planner_segments = 0;
result->datalocalityInfo = makeStringInfo();
/* fake data locality */
@@ -3995,7 +4016,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = -1;
+ result->planner_segments = 0;
return result;
}
@@ -4020,6 +4041,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
* 5 data size of random "from" relation
*/
+ udf_collector_context udf_context;
+ udf_context.udf_exist = false;
+
+ find_udf(query, &udf_context);
+ isTableFunctionExists = udf_context.udf_exist;
/*convert range table list to oid list and check whether table function exists
*we keep a full range table list and a range table list without result relation separately
*/
@@ -4046,7 +4072,15 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
/*use inherit resource*/
if (resourceLife == QRL_INHERIT) {
- resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+
+ if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
+ {
+ resource = NULL;
+ }
+ else
+ {
+ resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+ }
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4197,6 +4231,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
maxTargetSegmentNumber = enforce_virtual_segment_number;
minTargetSegmentNumber = enforce_virtual_segment_number;
}
+ /* in PBE mode, the execute should use the same vseg number. */
+ if(fixedVsegNum > 0 ){
+ maxTargetSegmentNumber = fixedVsegNum;
+ minTargetSegmentNumber = fixedVsegNum;
+ }
uint64_t before_rm_allocate_resource = gettime_microsec();
/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4204,9 +4243,40 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
mincost <<= 20;
int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
if (QRL_NONE != resourceLife) {
- resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
- maxTargetSegmentNumber, minTargetSegmentNumber,
- context.host_context.hostnameVolInfos, context.host_context.size);
+
+ if (SPI_IsInPrepare())
+ {
+ resource = NULL;
+ /*
+ * prepare need to get resource quota from RM
+ * and pass quota(planner_segments) to Orca or Planner to generate plan
+ * the following executes(in PBE) should reallocate the same number
+ * of resources.
+ */
+ uint32 seg_num;
+ uint32 seg_num_min;
+ uint32 seg_memory_mb;
+ double seg_core;
+
+ GetResourceQuota(maxTargetSegmentNumber,
+ minTargetSegmentNumber,
+ &seg_num,
+ &seg_num_min,
+ &seg_memory_mb,
+ &seg_core);
+
+ planner_segments = seg_num;
+ minTargetSegmentNumber = planner_segments;
+ maxTargetSegmentNumber = planner_segments;
+ }
+ else
+ {
+ resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+ maxTargetSegmentNumber,
+ minTargetSegmentNumber,
+ context.host_context.hostnameVolInfos,
+ context.host_context.size);
+ }
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4236,7 +4306,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
if (resource == NULL) {
result->resource = NULL;
- result->resource_parameters = NULL;
+ result->resource_parameters = resource_parameters;
result->alloc_results = NIL;
result->relsType = NIL;
result->planner_segments = planner_segments;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 07812aa..8e7645c 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,6 +46,7 @@
#include "cdb/memquota.h"
#include "executor/nodeFunctionscan.h"
#include "nodes/stack.h"
+#include "cdb/cdbdatalocality.h"
extern char *savedSeqServerHost;
extern int savedSeqServerPort;
@@ -74,6 +75,7 @@ static int _SPI_curid = -1;
static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
static char *_QD_currently_prepared_stmt = NULL;
+static int SPI_prepare_depth = 0;
static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
@@ -106,6 +108,32 @@ static bool _SPI_checktuples(void);
/* =================== interface functions =================== */
+bool SPI_IsInPrepare(void)
+{
+ if (SPI_prepare_depth > 0)
+ {
+ return true;
+ }
+ else if (SPI_prepare_depth < 0)
+ {
+ elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
+ SPI_prepare_depth);
+ }
+
+ return false;
+}
+
+void SPI_IncreasePrepareDepth(void)
+{
+ SPI_prepare_depth++;
+}
+
+void SPI_DecreasePrepareDepth(void)
+{
+ SPI_prepare_depth--;
+}
+
+
int
SPI_connect(void)
{
@@ -566,6 +594,8 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
_SPI_plan plan;
_SPI_plan *result;
+ SPI_IncreasePrepareDepth();
+
if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
{
SPI_result = SPI_ERROR_ARGUMENT;
@@ -591,9 +621,13 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
/* copy plan to procedure context */
result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
+
+ SPI_DecreasePrepareDepth();
}
PG_CATCH();
{
+ SPI_DecreasePrepareDepth();
+
_SPI_end_call(true);
PG_RE_THROW();
}
@@ -1819,13 +1853,37 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
(stmt->resource == NULL) &&
(stmt->resource_parameters != NULL) )
{
- stmt->resource = AllocateResource(stmt->resource_parameters->life,
- stmt->resource_parameters->slice_size,
- stmt->resource_parameters->iobytes,
- stmt->resource_parameters->max_target_segment_num,
- stmt->resource_parameters->min_target_segment_num,
- stmt->resource_parameters->vol_info,
- stmt->resource_parameters->vol_info_size);
+ SplitAllocResult *allocResult = NULL;
+
+ /* If this is a parallel plan. */
+ if (stmt->planTree->dispatch == DISPATCH_PARALLEL)
+ {
+ /*
+ * Now, we want to allocate resource.
+ */
+ allocResult = calculate_planner_segment_num(queryTree,
+ stmt->resource_parameters->life,
+ stmt->rtable,
+ stmt->intoPolicy,
+ stmt->nMotionNodes + stmt->nInitPlans + 1,
+ stmt->resource_parameters->min_target_segment_num);
+
+ Assert(allocResult);
+
+ if(stmt->resource !=NULL)
+ {
+ pfree(stmt->resource);
+ }
+ stmt->resource = allocResult->resource;
+ if(stmt->scantable_splits !=NULL)
+ {
+ list_free_deep(stmt->scantable_splits);
+ }
+ stmt->scantable_splits = allocResult->alloc_results;
+ stmt->planner_segments = allocResult->planner_segments;
+ stmt->datalocalityInfo = allocResult->datalocalityInfo;
+ pfree(allocResult);
+ }
}
originalStmt->resource = NULL;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index be057e0..c6be59a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,6 +91,8 @@ planner_hook_type planner_hook = NULL;
ParamListInfo PlannerBoundParamList = NULL; /* current boundParams */
+static int PlanningDepth = 0; /* Planning depth */
+
/* Expression kind codes for preprocess_expression */
#define EXPRKIND_QUAL 0
#define EXPRKIND_TARGET 1
@@ -165,6 +167,32 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
bool is_dummy_plan(Plan *plan);
+
+bool is_in_planning_phase(void)
+{
+ if (PlanningDepth > 0)
+ {
+ return true;
+ }
+ else if (PlanningDepth < 0)
+ {
+ elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
+ }
+
+ return false;
+}
+
+void increase_planning_depth(void)
+{
+ PlanningDepth++;
+}
+
+void decrease_planning_depth(void)
+{
+ PlanningDepth--;
+}
+
+
#ifdef USE_ORCA
/**
* Logging of optimization outcome
@@ -285,7 +313,7 @@ planner(Query *parse, int cursorOptions,
PlannedStmt *result = NULL;
instr_time starttime, endtime;
ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
- SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
+ SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
ppResult->saResult = initResult;
ppResult->stmt = NULL;
static int plannerLevel = 0;
@@ -300,6 +328,8 @@ planner(Query *parse, int cursorOptions,
* resource to run this query. After gaining the resource, we can perform the
* actual optimization.
*/
+ increase_planning_depth();
+
plannerLevel++;
if (!resourceNegotiateDone)
{
@@ -309,6 +339,8 @@ planner(Query *parse, int cursorOptions,
{
resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
+ decrease_planning_depth();
+
if(ppResult->stmt && ppResult->stmt->planTree)
{
isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -318,6 +350,8 @@ planner(Query *parse, int cursorOptions,
}
PG_CATCH();
{
+ decrease_planning_depth();
+
if ((ppResult != NULL))
{
pfree(ppResult);
@@ -430,7 +464,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = -1;
+ gp_segments_for_planner = 0;
}
SetActiveQueryResource(savedQueryResource);
if ((ppResult != NULL))
@@ -450,7 +484,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = -1;
+ gp_segments_for_planner = 0;
}
SetActiveQueryResource(savedQueryResource);
@@ -470,6 +504,7 @@ planner(Query *parse, int cursorOptions,
return result;
}
+
/*
* The new framework for HAWQ 2.0 query optimizer
*/
@@ -480,6 +515,8 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
PlannedStmt *plannedstmt = NULL;
do
{
+ udf_collector_context udf_context;
+ udf_context.udf_exist = false;
SplitAllocResult *allocResult = NULL;
Query *my_parse = copyObject(parse);
ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -495,12 +532,24 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
*/
allocResult = calculate_planner_segment_num(my_parse, resourceLife,
plannedstmt->rtable, plannedstmt->intoPolicy,
- plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
+ plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
+ -1);
Assert(allocResult);
(*result)->saResult = *allocResult;
pfree(allocResult);
+ }else{
+ find_udf(my_parse, &udf_context);
+ if(udf_context.udf_exist){
+ if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
+ int64 mincost = min_cost_for_each_query;
+ mincost <<= 20;
+ int avgSliceNum = 3;
+ (*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
+ GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
+ }
+ }
}
} while (0);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a80737f..a3d67f2 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,8 +2831,15 @@ simplify_function(Oid funcid, Oid result_type, List *args,
if (!HeapTupleIsValid(func_tuple))
elog(ERROR, "cache lookup failed for function %u", funcid);
- newexpr = evaluate_function(funcid, result_type, args,
- func_tuple, context);
+ if (is_in_planning_phase())
+ {
+ newexpr = NULL;
+ }
+ else
+ {
+ newexpr = evaluate_function(funcid, result_type, args,
+ func_tuple, context);
+ }
if (large_const(newexpr, context->max_size))
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index 6f1ee51..a8222fb 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,6 +162,7 @@ expression_tree_walker(Node *node,
case T_PartBoundExpr:
case T_PartBoundInclusionExpr:
case T_PartBoundOpenExpr:
+ case T_RangeTblEntry :
/* primitive node types with no expression subnodes */
break;
case T_Aggref:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 95f2bd5..67375d7 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -49,6 +49,13 @@ typedef struct SplitAllocResult
} SplitAllocResult;
/*
+ * structure containing all relation range table entries.
+ */
+typedef struct udf_collector_context {
+ bool udf_exist;
+} udf_collector_context;
+
+/*
* structure containing rel and type when execution
*/
typedef struct CurrentRelType {
@@ -86,7 +93,17 @@ void saveQueryResourceParameters(
* we calculate the appropriate planner segment_num.
*/
SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *rtable, GpPolicy *intoPolicy, int sliceNum);
+ List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
+
+/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node, udf_collector_context *context);
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context);
FILE *fp;
FILE *fpaoseg;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 867daf5..7b4ba1b 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,4 +155,11 @@ extern uint64 SPI_GetMemoryReservation(void);
extern void SPI_ReserveMemory(uint64 mem_reserved);
extern bool SPI_IsMemoryReserved(void);
+/**
+ * Query resource related routines.
+ */
+extern bool SPI_IsInPrepare(void);
+extern void SPI_IncreasePrepareCounter(void);
+extern void SPI_DecreasePrepareCounter(void);
+
#endif /* SPI_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index 48de348..cc2eb7d 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,4 +50,8 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
double dNumGroups,
AggClauseCounts *agg_counts);
+extern bool is_in_planning_phase(void);
+extern void increase_planning_depth(void);
+extern void decrease_planning_depth(void);
+
#endif /* PLANNER_H */
[32/50] incubator-hawq git commit: HAWQ-554. Create web external
table shouldnot omit ON clause.
Posted by sh...@apache.org.
HAWQ-554. Create web external table shouldnot omit ON clause.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17af44cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17af44cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17af44cb
Branch: refs/heads/HAWQ-459
Commit: 17af44cb139ddbe6b0e903d041d1177b73d2e556
Parents: 2513ad1
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:33:46 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800
----------------------------------------------------------------------
src/backend/parser/gram.y | 9 ++++++---
src/test/regress/output/exttab1.source | 6 ++++--
src/test/regress/output/exttab1_optimizer.source | 6 ++++--
3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/backend/parser/gram.y
----------------------------------------------------------------------
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 4d6f76f..dc0e13b 100755
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -4610,9 +4610,12 @@ CreateExternalStmt: CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n
/* if no ON clause specified, default to "ON ALL" */
if(extdesc->on_clause == NIL)
- {
- extdesc->on_clause = lappend(extdesc->on_clause,
- makeDefElem("all", (Node *)makeInteger(TRUE)));
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("ON clause must be specified in external table"),
+ errhint("Use ON number instread"),
+ errOmitLocation(true)));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/exttab1.source b/src/test/regress/output/exttab1.source
index bd9dec8..ec8ed3b 100755
--- a/src/test/regress/output/exttab1.source
+++ b/src/test/regress/output/exttab1.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
-- test for exec child process stderr showing in error message
--
create external web table ext_stderr1(a text) execute 'nosuchcommand' format 'text';
-ERROR: the ON ALL syntax for external tables is deprecated
+ERROR: ON clause must be specified in external table
+HINT: Use ON number instread
create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' format 'text';
-ERROR: the ON ALL syntax for external tables is deprecated
+ERROR: ON clause must be specified in external table
+HINT: Use ON number instread
--
-- bad csv (quote must be a single char)
--
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1_optimizer.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/exttab1_optimizer.source b/src/test/regress/output/exttab1_optimizer.source
index 9519769..3f03143 100755
--- a/src/test/regress/output/exttab1_optimizer.source
+++ b/src/test/regress/output/exttab1_optimizer.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
-- test for exec child process stderr showing in error message
--
create external web table ext_stderr1(a text) execute 'nosuchcommand' format 'text';
-ERROR: the ON ALL syntax for external tables is deprecated
+ERROR: ON clause must be specified in external table
+HINT: Use ON number instread
create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' format 'text';
-ERROR: the ON ALL syntax for external tables is deprecated
+ERROR: ON clause must be specified in external table
+HINT: Use ON number instread
--
-- bad csv (quote must be a single char)
--
[42/50] incubator-hawq git commit: HAWQ-561. Wrongly check variance
of number of virtual segments
Posted by sh...@apache.org.
HAWQ-561. Wrongly check variance of number of virtual segments
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1d9eb5e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1d9eb5e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1d9eb5e3
Branch: refs/heads/HAWQ-459
Commit: 1d9eb5e32a9c0a6a46e39f854b40442b5b0d12c8
Parents: 86d9b0c
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 19:18:39 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 19:18:39 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resourcepool.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1d9eb5e3/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index e5d083c..acc8dbe 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2556,7 +2556,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
Assert(minnvseg <= maxnvseg);
- if ( maxnvseg - minnvseg > rm_tolerate_nseg_limit )
+ if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_limit )
{
elog(LOG, "Reject virtual segment allocation based on data "
"locality information. After tentative allocation "
@@ -2565,7 +2565,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
"is %d, tolerated difference limit is %d.",
maxnvseg,
minnvseg,
- rm_tolerate_nseg_limit);
+ rm_nvseg_variance_among_seg_limit);
/* Return the allocated resource. */
List *vsegcntlist = NULL;
[12/50] incubator-hawq git commit: HAWQ-541. When YARN containers
disappear from container report, the segment available resource maybe negative
Posted by sh...@apache.org.
HAWQ-541. When YARN containers disappear from container report, the segment available resource maybe negative
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/99a307b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/99a307b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/99a307b1
Branch: refs/heads/HAWQ-459
Commit: 99a307b185ce0d3821a736ab39855eb4ee08541f
Parents: 5152fb0
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Mar 15 14:30:05 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Mar 15 14:30:05 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resourcepool.c | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/99a307b1/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 6a36801..d3a01a4 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2396,7 +2396,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
int segcountact = containerset == NULL ?
0 :
containerset->Available.MemoryMB / memory;
- if ( segcountact == 0 )
+ if ( segcountact <= 0 )
{
elog(RMLOG, "Segment %s does not have more resource to allocate. "
"This segment is skipped.",
@@ -3838,16 +3838,12 @@ void checkGRMContainerStatus(RB_GRMContainerStat ctnstats, int size)
Assert( segres->Allocated.MemoryMB >= 0 );
Assert( segres->Allocated.Core >= 0 );
- Assert( segres->Available.MemoryMB >= 0 );
- Assert( segres->Available.Core >= 0 );
minusResourceBundleData(&(ctnset->Allocated), ctn->MemoryMB, ctn->Core);
minusResourceBundleData(&(ctnset->Available), ctn->MemoryMB, ctn->Core);
Assert( ctnset->Allocated.MemoryMB >= 0 );
Assert( ctnset->Allocated.Core >= 0 );
- Assert( ctnset->Available.MemoryMB >= 0 );
- Assert( ctnset->Available.Core >= 0 );
reorderSegResourceAllocIndex(segres, PQUEMGR->RatioReverseIndex[ridx]);
reorderSegResourceAvailIndex(segres, PQUEMGR->RatioReverseIndex[ridx]);
[02/50] incubator-hawq git commit: HAWQ-523. Remove dead code in
executormgr
Posted by sh...@apache.org.
HAWQ-523. Remove dead code in executormgr
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/29a9beb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/29a9beb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/29a9beb1
Branch: refs/heads/HAWQ-459
Commit: 29a9beb1df109815deebf3750c23e2386fa5377e
Parents: b3e2add
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 11:36:19 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Mon Mar 14 14:00:14 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/executormgr.c | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a9beb1/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 4d1351d..88dd32f 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -202,13 +202,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
struct DispatchTask *task,
struct DispatchSlice *slice)
{
- executor->state = QES_UNINIT;
- if (desc == NULL)
- {
- executor->health = QEH_ERROR;
- return false;
- }
+ Assert(desc != NULL);
+ executor->state = QES_UNINIT;
executor->desc = desc;
executor->state = QES_DISPATCHABLE;
executor->health = QEH_NA;
[34/50] incubator-hawq git commit: HAWQ-529. Allocate resource for
udf in resource negotiator.
Posted by sh...@apache.org.
HAWQ-529. Allocate resource for udf in resource negotiator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d997b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d997b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d997b57
Branch: refs/heads/HAWQ-459
Commit: 4d997b5714bb09ed9020a56acfce13b1d6fd637c
Parents: 2c033d9
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:14:42 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbdatalocality.c | 213 ++++++++++++++++++++----------
src/backend/executor/spi.c | 71 ++++++++--
src/backend/optimizer/plan/planner.c | 59 ++++++++-
src/backend/optimizer/util/clauses.c | 11 +-
src/backend/optimizer/util/walkers.c | 1 +
src/include/cdb/cdbdatalocality.h | 19 ++-
src/include/executor/spi.h | 7 +
src/include/optimizer/planner.h | 4 +
8 files changed, 294 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index eec87b4..3f8fcb5 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,6 +33,9 @@
#include "access/filesplit.h"
#include "access/parquetsegfiles.h"
#include "catalog/catalog.h"
+#include "catalog/catquery.h"
+#include "catalog/pg_inherits.h"
+#include "catalog/pg_proc.h"
#include "cdb/cdbdatalocality.h"
#include "cdb/cdbutil.h"
#include "cdb/cdbvars.h"
@@ -41,6 +44,7 @@
#include "utils/tqual.h"
#include "utils/memutils.h"
#include "executor/execdesc.h"
+#include "executor/spi.h"
#include "nodes/nodes.h"
#include "nodes/parsenodes.h"
#include "optimizer/walkers.h"
@@ -54,11 +58,7 @@
#include "catalog/pg_proc.h"
#include "postgres.h"
#include "resourcemanager/utils/hashtable.h"
-#include "catalog/pg_inherits.h"
-//#include "utils/misc/guc.c"
-#define PRONAME 1
-#define PROISAGG 5
/* We need to build a mapping from host name to host index */
extern bool optimizer; /* Enable the optimizer */
@@ -376,7 +376,7 @@ static Block_Host_Index * update_data_dist_stat(
static HostDataVolumeInfo *search_host_in_stat_context(
split_to_segment_mapping_context *context, char *hostname);
-static bool IsAggFunction(char* funcName);
+static bool IsBuildInFunction(Oid funcOid);
static bool allocate_hash_relation(Relation_Data* rel_data,
Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -638,39 +638,35 @@ static void collect_range_tables(Query *query, List* full_range_table,
/*
*
*/
-static bool IsAggFunction(char* funcName) {
- if (funcName == NULL) {
- return false;
- }
- Relation pg_proc_rel;
- TupleDesc pg_proc_dsc;
- HeapTuple tuple;
- SysScanDesc pg_proc_scan;
-
- pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
- pg_proc_dsc = RelationGetDescr(pg_proc_rel);
- ScanKeyData skey;
+static bool IsBuildInFunction(Oid foid) {
- ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
- F_NAMEEQ, CStringGetDatum(funcName));
-
- pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
- ActiveSnapshot, 1, &skey);
- while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+ cqContext *pcqCtx;
+ HeapTuple procedureTuple;
+ Form_pg_proc procedureStruct;
- bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
- systable_endscan(pg_proc_scan);
- heap_close(pg_proc_rel, AccessShareLock);
- if (isAgg) {
- return true;
- } else {
- return false;
- }
+ /*
+ * get the procedure tuple corresponding to the given function Oid
+ */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_proc "
+ " WHERE oid = :1 ",
+ ObjectIdGetDatum(foid)));
+
+ procedureTuple = caql_getnext(pcqCtx);
+
+ if (!HeapTupleIsValid(procedureTuple))
+ elog(ERROR, "cache lookup failed for function %u", foid);
+ procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
+ caql_endscan(pcqCtx);
+ /* we treat proc namespace = 11 to build in function.*/
+ if (procedureStruct->pronamespace == 11) {
+ return true;
+ } else {
+ return false;
}
- systable_endscan(pg_proc_scan);
- heap_close(pg_proc_rel, AccessShareLock);
- return true;
}
+
/*
*
*/
@@ -684,25 +680,6 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
foreach(old_lc, *range_tables)
{
RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
- if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
- *isTableFunctionExists = true;
- }
- if (entry->rtekind == RTE_SUBQUERY) {
- Query* subQuery = entry->subquery;
- ListCell *lc;
- foreach(lc, subQuery->targetList)
- {
- TargetEntry *te = (TargetEntry *) lfirst(lc);
- bool isAggFunc = IsAggFunction(te->resname);
- // if target list of subquery contains non aggregate function,
- // then we consider the query contains and use default_segment_num guc
- // as the number of virtual segment
- if (!isAggFunc) {
- *isTableFunctionExists = true;
- }
- }
-
- }
if (entry->rtekind != RTE_RELATION) {
continue;
}
@@ -1833,7 +1810,7 @@ static int select_random_host_algorithm(Relation_Assignment_Context *context,
}
if (debug_fake_datalocality) {
fprintf(fp,
- "cur_size_of_whole_query is:"INT64_FORMAT", avg_size_of_whole_query is: %.3f",
+ "cur_size_of_whole_query is:%.0f, avg_size_of_whole_query is: %.3f",
context->totalvols_with_penalty[minindex] + net_disk_ratio * splitsize,
context->avg_size_of_whole_query);
}
@@ -2393,7 +2370,7 @@ static Relation_File** change_file_order_based_on_continuity(
Relation_Data *rel_data, TargetSegmentIDMap* idMap, int host_num,
int* fileCount, Relation_Assignment_Context *assignment_context) {
- Relation_File** file_vector;
+ Relation_File** file_vector = NULL;
int* isBlockContinue = (int *) palloc(sizeof(int) * host_num);
for (int i = 0; i < host_num; i++) {
isBlockContinue[i] = 0;
@@ -3429,45 +3406,45 @@ static void print_datalocality_overall_log_information(SplitAllocResult *result,
if(log_context->minSegmentNumofHost > 0 ){
fprintf(fpratio, "segmentnumber_perhost_max/min=%.2f\n", (double)(log_context->maxSegmentNumofHost / log_context->minSegmentNumofHost));
}else{
- fprintf(fpratio, "segmentnumber_perhost_max/min="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segmentnumber_perhost_max/min=%lld\n", INT64_MAX);
}
if(log_context->avgSegmentNumofHost > 0 ){
fprintf(fpratio, "segmentnumber_perhost_max/avg=%.2f\n", (double)(log_context->maxSegmentNumofHost / log_context->avgSegmentNumofHost));
}else{
- fprintf(fpratio, "segmentnumber_perhost_max/avg="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segmentnumber_perhost_max/avg=%lld\n", INT64_MAX);
}
if (log_context->minSizeSegmentOverall > 0){
fprintf(fpratio, "segments_size_max/min=%.5f\n", (double)log_context->maxSizeSegmentOverall / (double)log_context->minSizeSegmentOverall);
}else{
- fprintf(fpratio, "segments_size_max/min="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segments_size_max/min=%lld\n", INT64_MAX);
}
if (log_context->avgSizeOverall > 0){
fprintf(fpratio, "segments_size_max/avg=%.5f\n", log_context->maxSizeSegmentOverall / log_context->avgSizeOverall);
}else{
- fprintf(fpratio, "segments_size_max/avg="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segments_size_max/avg=%lld\n", INT64_MAX);
}
if (log_context->minSizeSegmentOverallPenalty > 0){
fprintf(fpratio, "segments_size_penalty_max/min=%.5f\n",(double)log_context->maxSizeSegmentOverallPenalty / (double)log_context->minSizeSegmentOverallPenalty);
}else{
- fprintf(fpratio, "segments_size_penalty_max/min="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segments_size_penalty_max/min=%lld\n", INT64_MAX);
}
if (log_context->avgSizeOverallPenalty > 0){
fprintf(fpratio, "segments_size_penalty_max/avg=%.5f\n",log_context->maxSizeSegmentOverallPenalty / log_context->avgSizeOverallPenalty);
}else{
- fprintf(fpratio, "segments_size_penalty_max/avg="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "segments_size_penalty_max/avg=%lld\n", INT64_MAX);
}
if (log_context->minContinuityOverall > 0){
fprintf(fpratio, "continuity_max/min=%.5f\n",log_context->maxContinuityOverall / log_context->minContinuityOverall);
}else{
- fprintf(fpratio, "continuity_max/min="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "continuity_max/min=%lld\n", INT64_MAX);
}
if (log_context->avgContinuityOverall > 0){
fprintf(fpratio, "continuity_max/avg=%.5f\n",log_context->maxContinuityOverall / log_context->avgContinuityOverall);
}else{
- fprintf(fpratio, "continuity_max/avg="INT64_FORMAT"\n", INT64_MAX);
+ fprintf(fpratio, "continuity_max/avg=%lld\n", INT64_MAX);
}
fflush(fpratio);
fclose(fpratio);
@@ -3976,11 +3953,54 @@ static void cleanup_allocation_algorithm(
}
/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,
+ udf_collector_context *context) {
+ if (node == NULL) {
+ return false;
+ }
+
+ if (IsA(node, Query)) {
+ return query_tree_walker((Query *) node, udf_collector_walker,
+ (void *) context,
+ QTW_EXAMINE_RTES);
+ }
+
+ /*For Aggref, we don't consider it as udf.*/
+
+ if(IsA(node,FuncExpr)){
+ if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
+ context->udf_exist = true;
+ }
+ return false;
+ }
+
+ return expression_tree_walker(node, udf_collector_walker,
+ (void *) context);
+
+ return false;
+}
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context) {
+
+ query_tree_walker(query, udf_collector_walker, (void *) context,
+ QTW_EXAMINE_RTES);
+
+ return;
+}
+
+
+/*
* calculate_planner_segment_num
+ * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
*/
SplitAllocResult *
calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
+ List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
SplitAllocResult *result = NULL;
QueryResource *resource = NULL;
QueryResourceParameters *resource_parameters = NULL;
@@ -3989,14 +4009,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
List *alloc_result = NIL;
split_to_segment_mapping_context context;
- int planner_segments = -1; /*virtual segments number for explain statement */
+ int planner_segments = 0; /*virtual segments number for explain statement */
result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
result->resource = NULL;
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = -1;
+ result->planner_segments = 0;
result->datalocalityInfo = makeStringInfo();
result->datalocalityTime = 0;
@@ -4014,7 +4034,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
result->resource_parameters = NULL;
result->alloc_results = NIL;
result->relsType = NIL;
- result->planner_segments = -1;
+ result->planner_segments = 0;
return result;
}
@@ -4039,6 +4059,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
* 5 data size of random "from" relation
*/
+ udf_collector_context udf_context;
+ udf_context.udf_exist = false;
+
+ find_udf(query, &udf_context);
+ isTableFunctionExists = udf_context.udf_exist;
/*convert range table list to oid list and check whether table function exists
*we keep a full range table list and a range table list without result relation separately
*/
@@ -4065,7 +4090,15 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
/*use inherit resource*/
if (resourceLife == QRL_INHERIT) {
- resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+
+ if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
+ {
+ resource = NULL;
+ }
+ else
+ {
+ resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+ }
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4216,6 +4249,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
maxTargetSegmentNumber = enforce_virtual_segment_number;
minTargetSegmentNumber = enforce_virtual_segment_number;
}
+ /* in PBE mode, the execute should use the same vseg number. */
+ if(fixedVsegNum > 0 ){
+ maxTargetSegmentNumber = fixedVsegNum;
+ minTargetSegmentNumber = fixedVsegNum;
+ }
uint64_t before_rm_allocate_resource = gettime_microsec();
/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4223,9 +4261,40 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
mincost <<= 20;
int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
if (QRL_NONE != resourceLife) {
- resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
- maxTargetSegmentNumber, minTargetSegmentNumber,
- context.host_context.hostnameVolInfos, context.host_context.size);
+
+ if (SPI_IsInPrepare())
+ {
+ resource = NULL;
+ /*
+ * prepare need to get resource quota from RM
+ * and pass quota(planner_segments) to Orca or Planner to generate plan
+ * the following executes(in PBE) should reallocate the same number
+ * of resources.
+ */
+ uint32 seg_num;
+ uint32 seg_num_min;
+ uint32 seg_memory_mb;
+ double seg_core;
+
+ GetResourceQuota(maxTargetSegmentNumber,
+ minTargetSegmentNumber,
+ &seg_num,
+ &seg_num_min,
+ &seg_memory_mb,
+ &seg_core);
+
+ planner_segments = seg_num;
+ minTargetSegmentNumber = planner_segments;
+ maxTargetSegmentNumber = planner_segments;
+ }
+ else
+ {
+ resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+ maxTargetSegmentNumber,
+ minTargetSegmentNumber,
+ context.host_context.hostnameVolInfos,
+ context.host_context.size);
+ }
saveQueryResourceParameters(
resource_parameters, /* resource_parameters */
@@ -4258,7 +4327,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
if (resource == NULL) {
result->resource = NULL;
- result->resource_parameters = NULL;
+ result->resource_parameters = resource_parameters;
result->alloc_results = NIL;
result->relsType = NIL;
result->planner_segments = planner_segments;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 07812aa..d0029fc 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,6 +46,7 @@
#include "cdb/memquota.h"
#include "executor/nodeFunctionscan.h"
#include "nodes/stack.h"
+#include "cdb/cdbdatalocality.h"
extern char *savedSeqServerHost;
extern int savedSeqServerPort;
@@ -74,6 +75,7 @@ static int _SPI_curid = -1;
static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
static char *_QD_currently_prepared_stmt = NULL;
+static int SPI_prepare_depth = 0;
static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
@@ -106,6 +108,32 @@ static bool _SPI_checktuples(void);
/* =================== interface functions =================== */
+bool SPI_IsInPrepare(void)
+{
+ if (SPI_prepare_depth > 0)
+ {
+ return true;
+ }
+ else if (SPI_prepare_depth < 0)
+ {
+ elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
+ SPI_prepare_depth);
+ }
+
+ return false;
+}
+
+void SPI_IncreasePrepareDepth(void)
+{
+ SPI_prepare_depth++;
+}
+
+void SPI_DecreasePrepareDepth(void)
+{
+ SPI_prepare_depth--;
+}
+
+
int
SPI_connect(void)
{
@@ -566,6 +594,8 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
_SPI_plan plan;
_SPI_plan *result;
+ SPI_IncreasePrepareDepth();
+
if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
{
SPI_result = SPI_ERROR_ARGUMENT;
@@ -591,9 +621,13 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
/* copy plan to procedure context */
result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
+
+ SPI_DecreasePrepareDepth();
}
PG_CATCH();
{
+ SPI_DecreasePrepareDepth();
+
_SPI_end_call(true);
PG_RE_THROW();
}
@@ -1112,6 +1146,20 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
qtlist = copyObject(qtlist);
ptlist = copyObject(ptlist);
+ PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
+
+ if ( (Gp_role == GP_ROLE_DISPATCH) &&
+ (stmt->resource_parameters != NULL) )
+ {
+ /*
+ * Now, we want to allocate resource.
+ */
+ stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
+ stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
+ stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
+ stmt->resource_parameters->vol_info_size);
+ }
+
/* If the plan has parameters, set them up */
if (spiplan->nargs > 0)
{
@@ -1812,22 +1860,21 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
* We only allocate resource for multiple executions of queries, NOT for utility commands.
* SELECT/INSERT are supported at present.
*/
- if( (queryTree->commandType == CMD_SELECT) ||
- (queryTree->commandType == CMD_INSERT) )
+ if((queryTree->commandType == CMD_SELECT) ||
+ (queryTree->commandType == CMD_INSERT))
{
- if ( (Gp_role == GP_ROLE_DISPATCH) &&
- (stmt->resource == NULL) &&
- (stmt->resource_parameters != NULL) )
+ if ((Gp_role == GP_ROLE_DISPATCH) &&
+ (stmt->resource == NULL) &&
+ (stmt->resource_parameters != NULL))
{
stmt->resource = AllocateResource(stmt->resource_parameters->life,
- stmt->resource_parameters->slice_size,
- stmt->resource_parameters->iobytes,
- stmt->resource_parameters->max_target_segment_num,
- stmt->resource_parameters->min_target_segment_num,
- stmt->resource_parameters->vol_info,
- stmt->resource_parameters->vol_info_size);
+ stmt->resource_parameters->slice_size,
+ stmt->resource_parameters->iobytes,
+ stmt->resource_parameters->max_target_segment_num,
+ stmt->resource_parameters->min_target_segment_num,
+ stmt->resource_parameters->vol_info,
+ stmt->resource_parameters->vol_info_size);
}
-
originalStmt->resource = NULL;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 6b2e13e..ebad24f 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,6 +91,8 @@ planner_hook_type planner_hook = NULL;
ParamListInfo PlannerBoundParamList = NULL; /* current boundParams */
+static int PlanningDepth = 0; /* Planning depth */
+
/* Expression kind codes for preprocess_expression */
#define EXPRKIND_QUAL 0
#define EXPRKIND_TARGET 1
@@ -165,6 +167,32 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
bool is_dummy_plan(Plan *plan);
+
+bool is_in_planning_phase(void)
+{
+ if (PlanningDepth > 0)
+ {
+ return true;
+ }
+ else if (PlanningDepth < 0)
+ {
+ elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
+ }
+
+ return false;
+}
+
+void increase_planning_depth(void)
+{
+ PlanningDepth++;
+}
+
+void decrease_planning_depth(void)
+{
+ PlanningDepth--;
+}
+
+
#ifdef USE_ORCA
/**
* Logging of optimization outcome
@@ -285,7 +313,7 @@ planner(Query *parse, int cursorOptions,
PlannedStmt *result = NULL;
instr_time starttime, endtime;
ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
- SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
+ SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
ppResult->saResult = initResult;
ppResult->stmt = NULL;
static int plannerLevel = 0;
@@ -300,6 +328,8 @@ planner(Query *parse, int cursorOptions,
* resource to run this query. After gaining the resource, we can perform the
* actual optimization.
*/
+ increase_planning_depth();
+
plannerLevel++;
if (!resourceNegotiateDone)
{
@@ -309,6 +339,8 @@ planner(Query *parse, int cursorOptions,
{
resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
+ decrease_planning_depth();
+
if(ppResult->stmt && ppResult->stmt->planTree)
{
isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -318,6 +350,8 @@ planner(Query *parse, int cursorOptions,
}
PG_CATCH();
{
+ decrease_planning_depth();
+
if ((ppResult != NULL))
{
pfree(ppResult);
@@ -430,7 +464,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = -1;
+ gp_segments_for_planner = 0;
}
SetActiveQueryResource(savedQueryResource);
if ((ppResult != NULL))
@@ -450,7 +484,7 @@ planner(Query *parse, int cursorOptions,
}
else
{
- gp_segments_for_planner = -1;
+ gp_segments_for_planner = 0;
}
SetActiveQueryResource(savedQueryResource);
@@ -471,6 +505,7 @@ planner(Query *parse, int cursorOptions,
return result;
}
+
/*
* The new framework for HAWQ 2.0 query optimizer
*/
@@ -481,6 +516,8 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
PlannedStmt *plannedstmt = NULL;
do
{
+ udf_collector_context udf_context;
+ udf_context.udf_exist = false;
SplitAllocResult *allocResult = NULL;
Query *my_parse = copyObject(parse);
ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -496,12 +533,26 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
*/
allocResult = calculate_planner_segment_num(my_parse, resourceLife,
plannedstmt->rtable, plannedstmt->intoPolicy,
- plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
+ plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
+ -1);
Assert(allocResult);
(*result)->saResult = *allocResult;
pfree(allocResult);
+ }else{
+ find_udf(my_parse, &udf_context);
+ if(udf_context.udf_exist){
+ if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
+ int64 mincost = min_cost_for_each_query;
+ mincost <<= 20;
+ int avgSliceNum = 3;
+ (*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
+ GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
+ }else{
+ (*result)->saResult.resource = AllocateResource(resourceLife, 0, 0, 0, 0, NULL, 0);
+ }
+ }
}
} while (0);
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a80737f..a3d67f2 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,8 +2831,15 @@ simplify_function(Oid funcid, Oid result_type, List *args,
if (!HeapTupleIsValid(func_tuple))
elog(ERROR, "cache lookup failed for function %u", funcid);
- newexpr = evaluate_function(funcid, result_type, args,
- func_tuple, context);
+ if (is_in_planning_phase())
+ {
+ newexpr = NULL;
+ }
+ else
+ {
+ newexpr = evaluate_function(funcid, result_type, args,
+ func_tuple, context);
+ }
if (large_const(newexpr, context->max_size))
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index 6f1ee51..a8222fb 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,6 +162,7 @@ expression_tree_walker(Node *node,
case T_PartBoundExpr:
case T_PartBoundInclusionExpr:
case T_PartBoundOpenExpr:
+ case T_RangeTblEntry :
/* primitive node types with no expression subnodes */
break;
case T_Aggref:
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index d28fc3e..224cb53 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -50,6 +50,13 @@ typedef struct SplitAllocResult
} SplitAllocResult;
/*
+ * structure containing all relation range table entries.
+ */
+typedef struct udf_collector_context {
+ bool udf_exist;
+} udf_collector_context;
+
+/*
* structure containing rel and type when execution
*/
typedef struct CurrentRelType {
@@ -87,7 +94,17 @@ void saveQueryResourceParameters(
* we calculate the appropriate planner segment_num.
*/
SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
- List *rtable, GpPolicy *intoPolicy, int sliceNum);
+ List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
+
+/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node, udf_collector_context *context);
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context);
FILE *fp;
FILE *fpaoseg;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 867daf5..7b4ba1b 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,4 +155,11 @@ extern uint64 SPI_GetMemoryReservation(void);
extern void SPI_ReserveMemory(uint64 mem_reserved);
extern bool SPI_IsMemoryReserved(void);
+/**
+ * Query resource related routines.
+ */
+extern bool SPI_IsInPrepare(void);
+extern void SPI_IncreasePrepareCounter(void);
+extern void SPI_DecreasePrepareCounter(void);
+
#endif /* SPI_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index 48de348..cc2eb7d 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,4 +50,8 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
double dNumGroups,
AggClauseCounts *agg_counts);
+extern bool is_in_planning_phase(void);
+extern void increase_planning_depth(void);
+extern void decrease_planning_depth(void);
+
#endif /* PLANNER_H */
[08/50] incubator-hawq git commit: HAWQ-473. Implement adding a entry
into gp_configuration_history and add a description column to show the reason
of this segment is down
Posted by sh...@apache.org.
HAWQ-473. Implement adding a entry into gp_configuration_history and
add a description column to show the reason of this segment is down
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d1780628
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d1780628
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d1780628
Branch: refs/heads/HAWQ-459
Commit: d178062876f0c55db7b1ac74946bd2ba096b779f
Parents: 2a15066
Author: Wen Lin <wl...@pivotal.io>
Authored: Tue Mar 15 10:00:16 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Mar 15 10:00:16 2016 +0800
----------------------------------------------------------------------
.../communication/rmcomm_RM2RMSEG.c | 28 +-
.../resourcemanager/include/resourcepool.h | 43 +-
src/backend/resourcemanager/requesthandler.c | 109 ++-
.../resourcemanager/requesthandler_RMSEG.c | 1 -
.../resourcebroker/resourcebroker_LIBYARN.c | 82 +-
.../resourcebroker_LIBYARN_proc.c | 1 -
src/backend/resourcemanager/resourcemanager.c | 91 +-
src/backend/resourcemanager/resourcepool.c | 857 ++++++++++++-------
src/backend/resourcemanager/resqueuemanager.c | 3 +-
src/backend/utils/gp/segadmin.c | 3 +-
src/include/catalog/gp_configuration.h | 27 +-
src/include/catalog/gp_segment_config.h | 11 +-
src/include/catalog/pg_proc.h | 5 +-
src/include/utils/builtins.h | 1 +
tools/bin/gppylib/data/2.0.json | 32 +-
15 files changed, 823 insertions(+), 471 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index fe4a595..ca332c9 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -231,11 +231,23 @@ void receivedRUAliveResponse(AsyncCommMessageHandlerContext context,
* This call makes resource pool remove unused containers.
*/
returnAllGRMResourceFromSegment(segres);
+
+ segres->Stat->StatusDesc |= SEG_STATUS_FAILED_PROBING_SEGMENT;
/* Set the host down in gp_segment_configuration table */
if (Gp_role != GP_ROLE_UTILITY)
{
+ SimpStringPtr description = build_segment_status_description(segres->Stat);
update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
- SEGMENT_STATUS_DOWN);
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ description->Str);
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
}
/* Set the host down. */
elog(WARNING, "Resource manager sets host %s from up to down "
@@ -281,10 +293,22 @@ void sentRUAliveError(AsyncCommMessageHandlerContext context)
* This call makes resource pool remove unused containers.
*/
returnAllGRMResourceFromSegment(segres);
+ segres->Stat->StatusDesc |= SEG_STATUS_COMMUNICATION_ERROR;
/* Set the host down in gp_segment_configuration table */
if (Gp_role != GP_ROLE_UTILITY)
{
- update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET, SEGMENT_STATUS_DOWN);
+ SimpStringPtr description = build_segment_status_description(segres->Stat);
+ update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ description->Str);
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
}
/* Set the host down. */
elog(LOG, "Resource manager sets host %s from up to down "
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index 6e1412c..5feedd3 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -145,13 +145,14 @@ struct SegStatData {
int32_t ID; /* Internal ID. */
uint16_t FailedTmpDirNum; /* Failed temporary directory number */
uint8_t FTSAvailable; /* If it is available now. */
- uint8_t GRMAvailable; /* If it is global resource available.*/
-
+ uint8_t GRMHandled; /* If its GRM status is handled */
uint32_t FTSTotalMemoryMB; /* FTS reports memory capacity. */
uint32_t FTSTotalCore; /* FTS reports core capacity. */
uint32_t GRMTotalMemoryMB; /* GRM reports memory capacity. */
uint32_t GRMTotalCore; /* GRM reports core capacity. */
uint64_t RMStartTimestamp; /* RM process reset timestamp */
+ uint32_t StatusDesc; /* Description of status */
+ uint32_t Reserved;
SegInfoData Info; /* 64-bit aligned. */
};
@@ -168,12 +169,9 @@ enum SegAvailabilityStatus {
};
int setSegStatHAWQAvailability( SegStat machine, uint8_t newstatus);
-int setSegStatGLOBAvailability( SegStat machine, uint8_t newstatus);
#define IS_SEGSTAT_FTSAVAILABLE(seg) \
((seg)->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
-#define IS_SEGSTAT_GRMAVAILABLE(seg) \
- ((seg)->GRMAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
/* Generate SegStat instance's report as a string saved in self maintained buffer. */
void generateSegStatReport(SegStat segstat, SelfMaintainBuffer buff);
@@ -610,7 +608,7 @@ int returnResourceToResourcePool(int memory,
void returnAllGRMResourceFromSegment(SegResource segres);
void dropAllGRMContainersFromSegment(SegResource segres);
-void returnAllGRMResourceFromGRMUnavailableSegments(void);
+void returnAllGRMResourceFromUnavailableSegments(void);
void generateSegResourceReport(int32_t nodeid, SelfMaintainBuffer buff);
@@ -632,7 +630,7 @@ int addOrderedResourceAvailTreeIndexByRatio(uint32_t ratio, BBST *tree);
int getOrderedResourceAvailTreeIndexByRatio(uint32_t ratio, BBST *tree);
int getOrderedResourceAllocTreeIndexByRatio(uint32_t ratio, BBST *tree);
-void setAllSegResourceGRMUnavailable(void);
+void setAllSegResourceGRMUnhandled(void);
void resetAllSegmentsGRMContainerFailAllocCount(void);
@@ -671,13 +669,11 @@ void adjustMemoryCoreValue(uint32_t *memorymb, uint32_t *core);
/* Clean up gp_segment_configuration */
void cleanup_segment_config(void);
+
+#define SEG_STATUS_DESCRIPTION_UP "segment is up"
/* update a segment's status in gp_segment_configuration table */
-void update_segment_status(int32_t id, char status);
-/* update a segment's status and failed temporary directory
- * in gp_segment_configuration table
- */
-void update_segment_failed_tmpdir
-(int32_t id, char status, int32_t failedNum, char* failedTmpDir);
+void update_segment_status(int32_t id, char status, char* description);
+
/* Add a new entry into gp_segment_configuration table*/
void add_segment_config_row(int32_t id,
char *hostname,
@@ -685,8 +681,25 @@ void add_segment_config_row(int32_t id,
uint32_t port,
char role,
char status,
- uint32_t failed_tmpdir_num,
- char* failed_tmpdir);
+ char* description);
+
+/*
+ * SegStatData's StatusDesc is a combination of below flags
+ */
+#define SEG_STATUS_HEARTBEAT_TIMEOUT 0x00000001
+#define SEG_STATUS_FAILED_PROBING_SEGMENT 0x00000002
+#define SEG_STATUS_COMMUNICATION_ERROR 0x00000004
+#define SEG_STATUS_FAILED_TMPDIR 0x00000008
+#define SEG_STATUS_RM_RESET 0x00000010
+#define SEG_STATUS_NO_GRM_NODE_REPORT 0x00000020
+
+/* Add a new entry into gp_configuration_history table */
+void add_segment_history_row(int32_t id,
+ char* hostname,
+ char* description);
+
+/* build a string of status description based on SegStat */
+SimpStringPtr build_segment_status_description(SegStat segstat);
/*
* In resource pool, segment's id starts from 0, however in gp_segment_configuration table,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index ba35e2e..a06d169 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -746,29 +746,13 @@ bool handleRMSEGRequestIMAlive(void **arg)
destroySelfMaintainBuffer(&newseginfo);
newsegstat->ID = SEGSTAT_ID_INVALID;
- newsegstat->GRMAvailable = RESOURCE_SEG_STATUS_UNSET;
RPCRequestHeadIMAlive header = SMBUFF_HEAD(RPCRequestHeadIMAlive,
&(conntrack->MessageBuff));
newsegstat->FailedTmpDirNum = header->TmpDirBrokenCount;
newsegstat->RMStartTimestamp = header->RMStartTimestamp;
-
- /*
- * Check if the there is any failed temporary directory on this segment.
- * if has, master considers this segment as down, even it has heart-beat report.
- */
- if (newsegstat->FailedTmpDirNum == 0)
- {
- newsegstat->FTSAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
- }
- else
- {
- elog(RMLOG, "Resource manager finds there is %d failed temporary directories "
- "on this segment, "
- "so mark this segment unavailable.",
- newsegstat->FailedTmpDirNum);
- newsegstat->FTSAvailable = RESOURCE_SEG_STATUS_UNAVAILABLE;
- }
+ newsegstat->StatusDesc = 0;
+ newsegstat->Reserved = 0;
bool capstatchanged = false;
if ( addHAWQSegWithSegStat(newsegstat, &capstatchanged) != FUNC_RETURN_OK )
@@ -977,13 +961,13 @@ bool handleRMRequestSegmentIsDown(void **arg)
while( (hostname - SMBUFF_CONTENT(&(conntrack->MessageBuff)) <
getSMBContentSize(&(conntrack->MessageBuff))) &&
- *hostname != '\0' )
+ *hostname != '\0' )
{
hostnamelen = strlen(hostname);
res = getSegIDByHostName(hostname, hostnamelen, &segid);
if ( res == FUNC_RETURN_OK )
{
- /* Get resourceinfo of the expected host. */
+ /* Get resource info of the expected host. */
SegResource segres = getSegResource(segid);
Assert( segres != NULL );
@@ -1002,45 +986,56 @@ bool handleRMRequestSegmentIsDown(void **arg)
else
{
elog(RMLOG, "Resource manager probes the status of host %s by "
- "sending RUAlive request.",
+ "sending RUAlive request.",
hostname);
- res = sendRUAlive(hostname);
- /* IN THIS CASE, the segment is considered as down. */
- if (res != FUNC_RETURN_OK)
- {
- /*----------------------------------------------------------
- * This call makes resource manager able to adjust queue and
- * mem/core trackers' capacity.
- *----------------------------------------------------------
- */
- setSegResHAWQAvailability(segres,
- RESOURCE_SEG_STATUS_UNAVAILABLE);
-
- /* Make resource pool remove unused containers */
- returnAllGRMResourceFromSegment(segres);
- /* Set the host down in gp_segment_configuration table */
- if (Gp_role != GP_ROLE_UTILITY)
- {
- update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
- SEGMENT_STATUS_DOWN);
- }
-
- /* Set the host down. */
- elog(LOG, "Resource manager sets host %s from up to down "
- "due to not reaching host.", hostname);
- }
- else
- {
- elog(RMLOG, "Resource manager triggered RUAlive request to "
- "host %s.",
+ res = sendRUAlive(hostname);
+ /* IN THIS CASE, the segment is considered as down. */
+ if (res != FUNC_RETURN_OK)
+ {
+ /*----------------------------------------------------------
+ * This call makes resource manager able to adjust queue and
+ * mem/core trackers' capacity.
+ *----------------------------------------------------------
+ */
+ setSegResHAWQAvailability(segres,
+ RESOURCE_SEG_STATUS_UNAVAILABLE);
+
+ /* Make resource pool remove unused containers */
+ returnAllGRMResourceFromSegment(segres);
+ /* Set the host down in gp_segment_configuration table */
+ segres->Stat->StatusDesc |= SEG_STATUS_FAILED_PROBING_SEGMENT;
+ if (Gp_role != GP_ROLE_UTILITY)
+ {
+ SimpStringPtr description = build_segment_status_description(segres->Stat);
+ update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ hostname,
+ description->Str);
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
+ }
+
+ /* Set the host down. */
+ elog(LOG, "Resource manager sets host %s from up to down "
+ "due to not reaching host.", hostname);
+ }
+ else
+ {
+ elog(RMLOG, "Resource manager triggered RUAlive request to "
+ "host %s.",
hostname);
- }
+ }
}
}
else {
elog(WARNING, "Resource manager cannot find host %s to check status, "
- "skip it.",
+ "skip it.",
hostname);
}
@@ -1124,7 +1119,7 @@ bool handleRMRequestTmpDir(void **arg)
RESPONSE_QD_TMPDIR);
elog(LOG, "Resource manager assigned temporary directory %s",
- tmpdir->Str);
+ tmpdir->Str);
}
conntrack->ResponseSent = false;
@@ -1271,11 +1266,11 @@ bool handleRMRequestDummy(void **arg)
sizeof(response),
conntrack->MessageMark1,
conntrack->MessageMark2,
- RESPONSE_DUMMY);
+ RESPONSE_DUMMY);
conntrack->ResponseSent = false;
- MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
- PCONTRACK->ConnToSend = lappend(PCONTRACK->ConnToSend, conntrack);
- MEMORY_CONTEXT_SWITCH_BACK
+ MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
+ PCONTRACK->ConnToSend = lappend(PCONTRACK->ConnToSend, conntrack);
+ MEMORY_CONTEXT_SWITCH_BACK
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/requesthandler_RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler_RMSEG.c b/src/backend/resourcemanager/requesthandler_RMSEG.c
index e667c1a..90946a0 100644
--- a/src/backend/resourcemanager/requesthandler_RMSEG.c
+++ b/src/backend/resourcemanager/requesthandler_RMSEG.c
@@ -182,7 +182,6 @@ int refreshLocalHostInstance(void)
RMSEG_INBUILDHOST->GRMTotalCore = 0.0;
RMSEG_INBUILDHOST->FTSTotalMemoryMB = DRMGlobalInstance->SegmentMemoryMB;
RMSEG_INBUILDHOST->FTSTotalCore = DRMGlobalInstance->SegmentCore;
- RMSEG_INBUILDHOST->GRMAvailable = RESOURCE_SEG_STATUS_UNSET;
RMSEG_INBUILDHOST->FTSAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
RMSEG_INBUILDHOST->ID = SEGSTAT_ID_INVALID;
RMSEG_INBUILDHOST->FailedTmpDirNum = failedTmpDirNum;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index 1e8c7ac..d446a6d 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -536,7 +536,9 @@ int handleRB2RM_ClusterReport(void)
uint32_t segsize;
int fd = ResBrokerNotifyPipe[0];
int piperes = 0;
- List *segstats = NULL;
+ List *segstats = NULL;
+ List *allsegres = NULL;
+ ListCell *cell = NULL;
PRESPOOL->RBClusterReportCounter++;
@@ -642,11 +644,7 @@ int handleRB2RM_ClusterReport(void)
return res;
}
- /*
- * Set all current segments GRM unavailable, only the segments identified by
- * one segment in segstats are available.
- */
- setAllSegResourceGRMUnavailable();
+ setAllSegResourceGRMUnhandled();
/*
* Start to update resource pool content. The YARN cluster total size is
@@ -683,6 +681,66 @@ int handleRB2RM_ClusterReport(void)
rm_pfree(PCONTEXT, segstat);
segstats = list_delete_first(segstats);
}
+
+ /*
+ * iterate all segments without GRM report,
+ * and update its status.
+ */
+ getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres);
+ foreach(cell, allsegres)
+ {
+ SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
+ bool statusDescChange = false;
+
+ /*
+ * skip segments handled in GRM report list
+ */
+ if (segres->Stat->GRMHandled)
+ continue;
+
+ /*
+ * Set no GRM node report flag for this segment.
+ */
+ if ((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT) == 0)
+ {
+ segres->Stat->StatusDesc |= SEG_STATUS_NO_GRM_NODE_REPORT;
+ statusDescChange = true;
+ }
+
+ if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
+ {
+ /*
+ * This segment is FTS available, but master hasn't
+ * gotten its GRM node report, so set this segment to DOWN.
+ */
+ setSegResHAWQAvailability(segres, RESOURCE_SEG_STATUS_UNAVAILABLE);
+ }
+
+ Assert(!IS_SEGSTAT_FTSAVAILABLE(segres->Stat));
+ if (statusDescChange && Gp_role != GP_ROLE_UTILITY)
+ {
+ SimpStringPtr description = build_segment_status_description(segres->Stat);
+ update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ description->Str);
+
+ elog(LOG, "Resource manager hasn't gotten GRM node report for segment(%s),"
+ "updates its status:'%c', description:%s",
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
+ }
+ }
+ freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
+
MEMORY_CONTEXT_SWITCH_BACK
elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
@@ -694,10 +752,10 @@ int handleRB2RM_ClusterReport(void)
PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
/*
- * If the segment is not GRM available, RM should return all containers
- * located upon them.
+ * If the segment is GRM unavailable or FTS unavailable,
+ * RM should return all containers located upon them.
*/
- returnAllGRMResourceFromGRMUnavailableSegments();
+ returnAllGRMResourceFromUnavailableSegments();
/* Refresh available node count. */
refreshAvailableNodeCount();
@@ -706,7 +764,7 @@ int handleRB2RM_ClusterReport(void)
PQUEMGR->GRMQueueCapacity = response.QueueCapacity;
PQUEMGR->GRMQueueCurCapacity = response.QueueCurCapacity;
PQUEMGR->GRMQueueMaxCapacity = (response.QueueMaxCapacity > 0 &&
- response.QueueMaxCapacity <= 1) ?
+ response.QueueMaxCapacity <= 1) ?
response.QueueMaxCapacity :
PQUEMGR->GRMQueueMaxCapacity;
PQUEMGR->GRMQueueResourceTight = response.ResourceTight > 0 ? true : false;
@@ -714,9 +772,9 @@ int handleRB2RM_ClusterReport(void)
refreshResourceQueueCapacity(false);
refreshActualMinGRMContainerPerSeg();
- PRESPOOL->LastUpdateTime = gettime_microsec();
+ PRESPOOL->LastUpdateTime = gettime_microsec();
- return FUNC_RETURN_OK;
+ return FUNC_RETURN_OK;
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
index 3afae7b..f99ed48 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
@@ -1482,7 +1482,6 @@ int RB2YARN_getClusterReport(DQueue hosts)
segstat->ID = SEGSTAT_ID_INVALID;
segstat->FTSAvailable = RESOURCE_SEG_STATUS_UNSET;
- segstat->GRMAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
segstat->GRMTotalMemoryMB = pnodereport->memoryCapability;
segstat->GRMTotalCore = pnodereport->vcoresCapability;
segstat->FTSTotalMemoryMB = 0;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 9a4bb85..138c5a0 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -489,7 +489,6 @@ int ResManagerMainServer2ndPhase(void)
PostPortNumber,
SEGMENT_ROLE_MASTER_CONFIG,
SEGMENT_STATUS_UP,
- 0,
"");
/* Load queue and user definition as no DDL now. */
@@ -2096,12 +2095,9 @@ int generateAllocRequestToBroker(void)
/*
* Resource manager skips this segment if
* 1) Not FTS available;
- * 2) Not GRM available;
- * 3) Having resource decrease pending.
+ * 2) Having resource decrease pending.
*/
if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
- (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
- !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
{
continue;
@@ -2311,12 +2307,9 @@ void completeAllocRequestToBroker(int32_t *reqmem,
/*
* Resource manager skips this segment if
* 1) Not FTS available;
- * 2) Not GRM available;
- * 3) Having resource decrease pending.
+ * 2) Having resource decrease pending.
*/
- if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
- (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
- !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
+ if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
{
index++;
@@ -2391,12 +2384,9 @@ void completeAllocRequestToBroker(int32_t *reqmem,
/*
* Resource manager skips this segment if
* 1) Not FTS available;
- * 2) Not GRM available;
- * 3) Having resource decrease pending.
+ * 2) Having resource decrease pending.
*/
if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
- (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
- !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
{
index++;
@@ -2566,31 +2556,51 @@ void updateStatusOfAllNodes()
curtime = gettime_microsec();
for(uint32_t idx = 0; idx < PRESPOOL->SegmentIDCounter; idx++)
{
- node = getSegResource(idx);
- if (node != NULL &&
- (curtime - node->LastUpdateTime >
+ node = getSegResource(idx);
+ uint8_t oldStatus = node->Stat->FTSAvailable;
+ if (node != NULL &&
+ (curtime - node->LastUpdateTime >
1000000LL * rm_segment_heartbeat_timeout) &&
- IS_SEGSTAT_FTSAVAILABLE(node->Stat) )
- {
- /*
- * This call makes resource manager able to adjust queue and mem/core
- * trackers' capacity.
- */
- setSegResHAWQAvailability(node, RESOURCE_SEG_STATUS_UNAVAILABLE);
- /*
- * This call makes resource pool remove unused containers.
- */
- returnAllGRMResourceFromSegment(node);
- if (Gp_role != GP_ROLE_UTILITY)
- {
- update_segment_status(idx + REGISTRATION_ORDER_OFFSET, SEGMENT_STATUS_DOWN);
- }
+ (node->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0)
+ {
+ /*
+ * This segment is heartbeat timeout, update its description
+ * and set it to unavailable if needed.
+ */
+ if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+ {
+ /*
+ * This call makes resource manager able to adjust queue and mem/core
+ * trackers' capacity.
+ */
+ setSegResHAWQAvailability(node, RESOURCE_SEG_STATUS_UNAVAILABLE);
+ /*
+ * This call makes resource pool remove unused containers.
+ */
+ returnAllGRMResourceFromSegment(node);
+ changedstatus = true;
+ }
- elog(WARNING, "Resource manager sets host %s from up to down.",
- GET_SEGRESOURCE_HOSTNAME(node));
+ node->Stat->StatusDesc |= SEG_STATUS_HEARTBEAT_TIMEOUT;
+ if (Gp_role != GP_ROLE_UTILITY)
+ {
+ SimpStringPtr description = build_segment_status_description(node->Stat);
+ update_segment_status(idx + REGISTRATION_ORDER_OFFSET,
+ SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(idx + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(node),
+ (description->Len > 0)?description->Str:"");
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
+ }
- changedstatus = true;
- }
+ elog(WARNING, "Resource manager sets host %s heartbeat timeout.",
+ GET_SEGRESOURCE_HOSTNAME(node));
+ }
}
if ( changedstatus )
@@ -2711,11 +2721,10 @@ int loadHostInformationIntoResourcePool(void)
/* Build machine info instance. */
SegStat segstat = (SegStat)rm_palloc0(PCONTEXT,
- offsetof(SegStatData, Info) +
- seginfobuff.Cursor + 1);
- segstat->ID = SEGSTAT_ID_INVALID;
- segstat->GRMAvailable = RESOURCE_SEG_STATUS_UNSET;
- segstat->FTSAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
+ offsetof(SegStatData, Info) +
+ seginfobuff.Cursor + 1);
+ segstat->ID = SEGSTAT_ID_INVALID;
+ segstat->FTSAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
segstat->FTSTotalMemoryMB = DRMGlobalInstance->SegmentMemoryMB;
segstat->FTSTotalCore = DRMGlobalInstance->SegmentCore;
segstat->GRMTotalMemoryMB = 0;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 329a5c6..6a36801 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -26,6 +26,8 @@
#include "resourcepool.h"
#include "gp-libpq-fe.h"
#include "gp-libpq-int.h"
+#include "utils/builtins.h"
+#include "catalog/pg_proc.h"
void addSegResourceAvailIndex(SegResource segres);
void addSegResourceAllocIndex(SegResource segres);
@@ -467,12 +469,99 @@ cleanup:
PQclear(result);
PQfinish(conn);
}
+
+/*
+ * Remove all entries in gp_configuration_history.
+ *
+ * gp_remove_segment_history()
+ *
+ * Returns:
+ * true upon success, otherwise throws error.
+ */
+Datum
+gp_remove_segment_history(PG_FUNCTION_ARGS)
+{
+ int libpqres = CONNECTION_OK;
+ PGconn *conn = NULL;
+ char conninfo[512];
+ PQExpBuffer sql = NULL;
+ PGresult* result = NULL;
+ int ret = true;
+
+ if (!superuser())
+ elog(ERROR, "gp_remove_segment_history can only be run by a superuser");
+
+ if (Gp_role != GP_ROLE_UTILITY)
+ elog(ERROR, "gp_remove_segment_history can only be run in utility mode");
+
+ sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
+ "dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
+ conn = PQconnectdb(conninfo);
+ if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
+ {
+ elog(WARNING, "Fail to connect database when cleanup "
+ "segment configuration catalog table, error code: %d, %s",
+ libpqres,
+ PQerrorMessage(conn));
+ PQfinish(conn);
+ PG_RETURN_BOOL(false);
+ }
+
+ result = PQexec(conn, "BEGIN");
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ "BEGIN",
+ PQresultErrorMessage(result));
+ ret = false;
+ goto cleanup;
+ }
+ PQclear(result);
+
+ sql = createPQExpBuffer();
+ appendPQExpBuffer(sql,"DELETE FROM gp_configuration_history");
+ result = PQexec(conn, sql->data);
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ sql->data,
+ PQresultErrorMessage(result));
+ ret = false;
+ goto cleanup;
+ }
+ PQclear(result);
+
+ result = PQexec(conn, "COMMIT");
+ if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+ {
+ elog(WARNING, "Fail to run SQL: %s when cleanup "
+ "segment history catalog table, reason : %s",
+ "COMMIT",
+ PQresultErrorMessage(result));
+ ret = false;
+ goto cleanup;
+ }
+
+ elog(LOG, "Cleanup segment history catalog table successfully!");
+
+cleanup:
+ if(sql)
+ destroyPQExpBuffer(sql);
+ if(result)
+ PQclear(result);
+ PQfinish(conn);
+
+ PG_RETURN_BOOL(ret);
+}
+
/*
* update a segment's status in gp_segment_configuration table.
* id : registration order of this segment
* status : new status of this segment
*/
-void update_segment_status(int32_t id, char status)
+void update_segment_status(int32_t id, char status, char* description)
{
int libpqres = CONNECTION_OK;
PGconn *conn = NULL;
@@ -480,15 +569,22 @@ void update_segment_status(int32_t id, char status)
PQExpBuffer sql = NULL;
PGresult* result = NULL;
+ if (status == SEGMENT_STATUS_UP)
+ Assert(strlen(description) == 0);
+ else if (status == SEGMENT_STATUS_DOWN)
+ Assert(strlen(description) != 0);
+ else
+ Assert(0);
+
sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
"dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
conn = PQconnectdb(conninfo);
if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
{
elog(WARNING, "Fail to connect database when update segment's status "
- "in segment configuration catalog table, error code: %d, %s",
- libpqres,
- PQerrorMessage(conn));
+ "in segment configuration catalog table, error code: %d, %s",
+ libpqres,
+ PQerrorMessage(conn));
PQfinish(conn);
return;
}
@@ -497,23 +593,24 @@ void update_segment_status(int32_t id, char status)
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
{
elog(WARNING, "Fail to run SQL: %s when update segment's status "
- "in segment configuration catalog table, reason : %s",
- "BEGIN",
- PQresultErrorMessage(result));
+ "in segment configuration catalog table, reason : %s",
+ "BEGIN",
+ PQresultErrorMessage(result));
goto cleanup;
}
PQclear(result);
sql = createPQExpBuffer();
- appendPQExpBuffer(sql,"UPDATE gp_segment_configuration SET status='%c' WHERE registration_order=%d",
- status, id);
+ appendPQExpBuffer(sql, "UPDATE gp_segment_configuration SET status='%c', "
+ "description='%s' WHERE registration_order=%d",
+ status, description, id);
result = PQexec(conn, sql->data);
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
{
elog(WARNING, "Fail to run SQL: %s when update segment's status "
- "in segment configuration catalog table, reason : %s",
- sql->data,
- PQresultErrorMessage(result));
+ "in segment configuration catalog table, reason : %s",
+ sql->data,
+ PQresultErrorMessage(result));
goto cleanup;
}
PQclear(result);
@@ -528,9 +625,10 @@ void update_segment_status(int32_t id, char status)
goto cleanup;
}
- elog(LOG, "Update a segment's status to '%c' in segment configuration catalog table,"
- "registration_order : %d",
- status, id);
+ elog(LOG, "Update a segment(registration_order:%d)'s status to '%c',"
+ "description to '%s', "
+ "in segment configuration catalog table,",
+ id, status, description);
cleanup:
if(sql)
@@ -541,19 +639,17 @@ cleanup:
}
/*
- * update a segment's status and failed tmp dir
- * in gp_segment_configuration table.
+ * Insert a row into gp_configuration_history,
+ * to record the status change of a segment.
* id : registration order of this segment
- * status : new status of this segment
- * failedNum : number of failed temporary directory
- * failedTmpDir : failed temporary directory list, separated by comma
+ * hostname : hostname of this segment
+ * description : description of status change
*/
-void update_segment_failed_tmpdir
-(int32_t id, char status, int32_t failedNum, char* failedTmpDir)
+void add_segment_history_row(int32_t id, char* hostname, char* description)
{
int libpqres = CONNECTION_OK;
PGconn *conn = NULL;
- char conninfo[512];
+ char conninfo[1024];
PQExpBuffer sql = NULL;
PGresult* result = NULL;
@@ -562,8 +658,8 @@ void update_segment_failed_tmpdir
conn = PQconnectdb(conninfo);
if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
{
- elog(WARNING, "Fail to connect database when update segment's failed tmpdir "
- "in segment configuration catalog table, error code: %d, %s",
+ elog(WARNING, "Fail to connect database when add a new row into "
+ "segment configuration history catalog table, error code: %d, %s",
libpqres,
PQerrorMessage(conn));
PQfinish(conn);
@@ -573,24 +669,28 @@ void update_segment_failed_tmpdir
result = PQexec(conn, "BEGIN");
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
{
- elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
- "in segment configuration catalog table, reason : %s",
+ elog(WARNING, "Fail to run SQL: %s when add a new row into "
+ "segment configuration history catalog table, reason : %s",
"BEGIN",
PQresultErrorMessage(result));
goto cleanup;
}
PQclear(result);
+ TimestampTz curtime = GetCurrentTimestamp();
+ const char *curtimestr = timestamptz_to_str(curtime);
sql = createPQExpBuffer();
- appendPQExpBuffer(sql, "UPDATE gp_segment_configuration SET "
- "status='%c', failed_tmpdir_num = '%d', failed_tmpdir = '%s' "
- "WHERE registration_order=%d",
- status, failedNum, failedTmpDir, id);
+ appendPQExpBuffer(sql,
+ "INSERT INTO gp_configuration_history"
+ "(time, registration_order, hostname, description) "
+ "VALUES ('%s','%d','%s','%s')",
+ curtimestr, id, hostname, description);
+
result = PQexec(conn, sql->data);
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
{
- elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
- "in segment configuration catalog table, reason : %s",
+ elog(WARNING, "Fail to run SQL: %s when add a new row into "
+ "segment configuration history catalog table, reason : %s",
sql->data,
PQresultErrorMessage(result));
goto cleanup;
@@ -600,18 +700,16 @@ void update_segment_failed_tmpdir
result = PQexec(conn, "COMMIT");
if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
{
- elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
- "in segment configuration catalog table, reason : %s",
+ elog(WARNING, "Fail to run SQL: %s when add a new row into "
+ "segment configuration history catalog table, reason : %s",
"COMMIT",
PQresultErrorMessage(result));
goto cleanup;
}
- elog(LOG, "Update a segment's failed tmpdir:"
- "status to '%c', failed_tmpdir_num to '%d', failed_tmpdir to '%s' "
- "in segment configuration catalog table,"
- "registration_order : %d",
- status, failedNum, failedTmpDir, id);
+ elog(LOG, "Add a new row into segment configuration history catalog table,"
+ "time: %s, registration order:%d, hostname:%s, description:%s",
+ curtimestr, id, hostname, description);
cleanup:
if(sql)
@@ -629,8 +727,7 @@ cleanup:
* port : port of this segment
* role : role of this segment
* status : up or down
- * failed_tmpdir_num : the number of failed temporary directory
- * failed_tmpdir : failed temporary directory, separated by comma
+ * description : the description of this segment's current status
*/
void add_segment_config_row(int32_t id,
char* hostname,
@@ -638,9 +735,7 @@ void add_segment_config_row(int32_t id,
uint32_t port,
char role,
char status,
- uint32_t
- failed_tmpdir_num,
- char* failed_tmpdir)
+ char* description)
{
int libpqres = CONNECTION_OK;
PGconn *conn = NULL;
@@ -677,10 +772,10 @@ void add_segment_config_row(int32_t id,
{
appendPQExpBuffer(sql,
"INSERT INTO gp_segment_configuration"
- "(registration_order,role,status,port,hostname,address,failed_tmpdir_num,failed_tmpdir) "
+ "(registration_order,role,status,port,hostname,address,description) "
"VALUES "
- "(%d,'%c','%c',%d,'%s','%s',%d,'%s')",
- id,role,status,port,hostname,address,failed_tmpdir_num,failed_tmpdir);
+ "(%d,'%c','%c',%d,'%s','%s','%s')",
+ id,role,status,port,hostname,address,description);
}
else
{
@@ -713,8 +808,8 @@ void add_segment_config_row(int32_t id,
elog(LOG, "Add a new row into segment configuration catalog table,"
"registration order:%d, role:%c, status:%c, port:%d, "
- "hostname:%s, address:%s, failed_tmpdir_num:%d, failed_tmpdir:%s",
- id, role, status, port, hostname, address, failed_tmpdir_num, failed_tmpdir);
+ "hostname:%s, address:%s, description:%s",
+ id, role, status, port, hostname, address, description);
cleanup:
if(sql)
@@ -758,41 +853,38 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
*/
hostname = GET_SEGINFO_HOSTNAME(&(segstat->Info));
hostnamelen = segstat->Info.HostNameLen;
- res = getSegIDByHostName(hostname, hostnamelen, &segid);
- if ( res != FUNC_RETURN_OK )
- {
- /* Try addresses of this machine. */
- for ( int i = 0 ; i < segstat->Info.HostAddrCount ; ++i )
- {
-
- elog(DEBUG5, "Resource manager checks host ip (%d)th to get segment.", i);
- AddressString addr = NULL;
- getSegInfoHostAddrStr(&(segstat->Info), i, &addr);
- if ( strcmp(addr->Address, IPV4_DOT_ADDR_LO) == 0 &&
- segstat->Info.HostAddrCount > 1 )
- {
- /*
- * If the host has only one address as 127.0.0.1, we have to
- * save it to track the only one segment, otherwise, we skip
- * 127.0.0.1 address in resource pool. Because, each node has
- * one entry of this address.
- */
- continue;
- }
+ res = getSegIDByHostName(hostname, hostnamelen, &segid);
+ if ( res != FUNC_RETURN_OK )
+ {
+ /* Try addresses of this machine. */
+ for ( int i = 0 ; i < segstat->Info.HostAddrCount ; ++i )
+ {
+ elog(DEBUG5, "Resource manager checks host ip (%d)th to get segment.", i);
+ AddressString addr = NULL;
+ getSegInfoHostAddrStr(&(segstat->Info), i, &addr);
+ if ( strcmp(addr->Address, IPV4_DOT_ADDR_LO) == 0 &&
+ segstat->Info.HostAddrCount > 1 )
+ {
+ /*
+ * If the host has only one address as 127.0.0.1, we have to
+ * save it to track the only one segment, otherwise, we skip
+ * 127.0.0.1 address in resource pool. Because, each node has
+ * one entry of this address.
+ */
+ continue;
+ }
- res = getSegIDByHostAddr((uint8_t *)(addr->Address), addr->Length, &segid);
- if ( res == FUNC_RETURN_OK )
- {
- break;
- }
- }
- }
+ res = getSegIDByHostAddr((uint8_t *)(addr->Address), addr->Length, &segid);
+ if ( res == FUNC_RETURN_OK )
+ {
+ break;
+ }
+ }
+ }
- /* CASE 1. It is a new host. */
+ /* CASE 1. It is a new host. */
if ( res != FUNC_RETURN_OK )
{
- uint8_t reportStatus = segstat->FTSAvailable;
-
/* Create machine information and corresponding resource information. */
segresource = createSegResource(segstat);
@@ -815,7 +907,7 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
setSimpleStringRef(&hostnamekey, hostname, hostnamelen);
oldval = setHASHTABLENode(&(PRESPOOL->SegmentHostNameIndexed),
- TYPCONVERT(void *, &hostnamekey),
+ TYPCONVERT(void *, &hostnamekey),
TYPCONVERT(void *, segid),
false /* There should be no old value. */);
if ( oldval != NULL )
@@ -844,22 +936,39 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
TYPCONVERT(void *, &hostaddrkey)) == NULL )
{
setHASHTABLENode( &(PRESPOOL->SegmentHostAddrIndexed),
- TYPCONVERT(void *, &hostaddrkey),
+ TYPCONVERT(void *, &hostaddrkey),
TYPCONVERT(void *, segid),
false /* There should be no old value. */);
elog(LOG, "Resource manager tracked ip address '%.*s' for host '%s'",
hostaddrkey.Len, hostaddrkey.Array,
- hostname);
+ hostname);
}
}
/*
- * This is a new host registration. Normally the status is available,
- * But if the number of failed temporary directory exceeds guc,
+ * If in GRM mode, the new registered segment is marked
+ * as DOWN, since master hasn't get a cluster report for it
+ * from global Resource Manager.
+ */
+ if (DRMGlobalInstance->ImpType != NONE_HAWQ2)
+ {
+ segresource->Stat->StatusDesc |= SEG_STATUS_NO_GRM_NODE_REPORT;
+ }
+
+ /*
+ * If there is any failed temporary directory,
* this segment is considered as unavailable.
*/
- setSegResHAWQAvailability(segresource, reportStatus);
+ if (segresource->Stat->FailedTmpDirNum != 0)
+ {
+ segresource->Stat->StatusDesc |= SEG_STATUS_FAILED_TMPDIR;
+ }
+
+ if (segresource->Stat->StatusDesc == 0)
+ {
+ setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_AVAILABLE);
+ }
/* Add this node into the table gp_segment_configuration */
AddressString straddr = NULL;
@@ -868,16 +977,26 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
if (Gp_role != GP_ROLE_UTILITY)
{
+ SimpStringPtr description = build_segment_status_description(segresource->Stat);
add_segment_config_row (segid+REGISTRATION_ORDER_OFFSET,
hostname,
straddr->Address,
segresource->Stat->Info.port,
SEGMENT_ROLE_PRIMARY,
- segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
+ IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
- segresource->Stat->FailedTmpDirNum,
- segresource->Stat->FailedTmpDirNum == 0 ?
- "":GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
+ (description->Len > 0)?description->Str:"");
+
+ add_segment_history_row(segid+REGISTRATION_ORDER_OFFSET,
+ hostname,
+ IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+ SEG_STATUS_DESCRIPTION_UP:description->Str);
+
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
}
if (segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
@@ -901,37 +1020,78 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
else {
segresource = getSegResource(segid);
Assert(segresource != NULL);
+ uint32_t oldStatusDesc = segresource->Stat->StatusDesc;
uint8_t oldStatus = segresource->Stat->FTSAvailable;
- bool statusChanged = oldStatus != segstat->FTSAvailable;
/*
- * Check if RM process is restarted in this segment.
* If the latest reported RM process startup timestamp doesn't
* match the previous, master RM consider segment's RM process
* has restarted.
* In rare case, the system's time is reset and segment's RM process
* happen to get a same timestamp with previous one.
*/
- if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp)
+ if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp &&
+ (segresource->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0 &&
+ (segresource->Stat->StatusDesc & SEG_STATUS_COMMUNICATION_ERROR) == 0 &&
+ (segresource->Stat->StatusDesc & SEG_STATUS_FAILED_PROBING_SEGMENT) == 0)
{
/*
- * This segment's RM process has restarted,
- * we should clean up old status, so mark it down.
+ * This segment's RM process has restarted.
+ * if StatusDesc doesn't have heartbeat timeout flag, or communication error,
+ * or failed probing segment flag, this segment is set to DOWN.
+ * It will be set to UP when reports a new heartbeat.
*/
- if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE && !statusChanged)
- {
- segstat->FTSAvailable = RESOURCE_SEG_STATUS_UNAVAILABLE;
- statusChanged = true;
- }
+ segresource->Stat->StatusDesc |= SEG_STATUS_RM_RESET;
segresource->Stat->RMStartTimestamp = segstat->RMStartTimestamp;
elog(LOG, "Master RM finds segment:%s 's RM process has restarted. "
- "old status:%d, new status:%d",
+ "old status:%d",
GET_SEGRESOURCE_HOSTNAME(segresource),
- oldStatus,
- segstat->FTSAvailable);
+ oldStatus);
+ }
+ else
+ {
+ if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp)
+ {
+ segresource->Stat->RMStartTimestamp = segstat->RMStartTimestamp;
+ }
+ /*
+ * Now clear heartbeat timeout flag, RM reset flag,
+ * no response flag, communication error flag
+ * and RM reset flag in StatusDesc
+ */
+ if ((segresource->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) != 0)
+ {
+ segresource->Stat->StatusDesc &= ~SEG_STATUS_HEARTBEAT_TIMEOUT;
+ elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+ "clear its heartbeat timeout flag",
+ GET_SEGRESOURCE_HOSTNAME(segresource));
+ }
+ if ((segresource->Stat->StatusDesc & SEG_STATUS_FAILED_PROBING_SEGMENT) != 0)
+ {
+ segresource->Stat->StatusDesc &= ~SEG_STATUS_FAILED_PROBING_SEGMENT;
+ elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+ "clear its failed probing segment flag",
+ GET_SEGRESOURCE_HOSTNAME(segresource));
+ }
+ if ((segresource->Stat->StatusDesc & SEG_STATUS_COMMUNICATION_ERROR) != 0)
+ {
+ segresource->Stat->StatusDesc &= ~SEG_STATUS_COMMUNICATION_ERROR;
+ elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+ "clear its communication error flag",
+ GET_SEGRESOURCE_HOSTNAME(segresource));
+ }
+ if ((segresource->Stat->StatusDesc & SEG_STATUS_RM_RESET) != 0)
+ {
+ segresource->Stat->StatusDesc &= ~SEG_STATUS_RM_RESET;
+ elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+ "clear its RM reset flag",
+ GET_SEGRESOURCE_HOSTNAME(segresource));
+ }
}
- /* Check if temporary directory path is changed */
+ /*
+ * If temporary directory path is changed, update SegStatData
+ */
bool tmpDirChanged = false;
if (segresource->Stat->FailedTmpDirNum != segstat->FailedTmpDirNum)
{
@@ -944,160 +1104,162 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)) != 0)
{
tmpDirChanged = true;
- elog(LOG, "Resource manager finds segment %s(%d) 's "
- "failed temporary directory is changed from "
- "'%s' to '%s'",
- GET_SEGRESOURCE_HOSTNAME(segresource),
- segid,
- GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
- GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
}
}
- /*
- * Either the FTSAvailable or the failed temporary directory
- * of this segment is changed.
- */
- if (statusChanged || tmpDirChanged)
+ if (tmpDirChanged)
{
- if (statusChanged && !tmpDirChanged)
- {
- if (Gp_role != GP_ROLE_UTILITY)
- {
- update_segment_status(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
- segstat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
- SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN);
- }
-
- setSegResHAWQAvailability(segresource, segstat->FTSAvailable);
- /*
- * Segment is set from up to down, return resource.
- */
- if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
- {
- *capstatchanged = true;
- returnAllGRMResourceFromSegment(segresource);
- }
+ elog(RMLOG, "Resource manager is going to set segment %s(%d) 's "
+ "failed temporary directory from "
+ "'%s' to '%s'",
+ GET_SEGRESOURCE_HOSTNAME(segresource),
+ segid,
+ segresource->Stat->FailedTmpDirNum == 0 ?
+ "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
+ segstat->FailedTmpDirNum == 0 ?
+ "" : GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
+
+ int old = segresource->Stat->Info.FailedTmpDirLen == 0 ?
+ 0 :__SIZE_ALIGN64(segresource->Stat->Info.FailedTmpDirLen+1);
+ int new = segstat->Info.FailedTmpDirLen == 0 ?
+ 0 : __SIZE_ALIGN64(segstat->Info.FailedTmpDirLen+1);
+
+ int current = segresource->Stat->Info.Size -
+ (segresource->Stat->Info.HostNameOffset + __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1));
+ if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
+ current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
+ if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
+ current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
- elog(LOG, "Master resource manager sets segment %s(%d)'s status "
- "to %c",
- GET_SEGRESOURCE_HOSTNAME(segresource),
- segid,
- segstat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
- SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN);
+ /*
+ * repalloc memory if new size exceeds the old one.
+ * we don't shrink memory size if new size is less than the old one.
+ */
+ if (new > old && current < new)
+ {
+ SegStat newSegStat = rm_repalloc(PCONTEXT,
+ segresource->Stat,
+ offsetof(SegStatData, Info) +
+ segresource->Stat->Info.Size + (new - old));
+ segresource->Stat = newSegStat;
+ segresource->Stat->Info.Size += (new - old);
}
- else
+
+ if (segresource->Stat->Info.FailedTmpDirOffset == 0)
{
- /*
- * Failed temporary directory is changed,
- * if the length of new failed temporary directory exceeds the old one,
- * we need to repalloc SegInfoData
- */
- elog(RMLOG, "Master resource manager is going to set segment %s(%d)'s "
- "failed temporary directory from '%s' to '%s'",
- GET_SEGRESOURCE_HOSTNAME(segresource),
- segid,
- segresource->Stat->FailedTmpDirNum == 0 ?
- "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
- segstat->FailedTmpDirNum == 0 ?
- "" : GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
-
- int old = segresource->Stat->Info.FailedTmpDirLen == 0 ?
- 0 :__SIZE_ALIGN64(segresource->Stat->Info.FailedTmpDirLen+1);
- int new = segstat->Info.FailedTmpDirLen == 0 ?
- 0 : __SIZE_ALIGN64(segstat->Info.FailedTmpDirLen+1);
-
- int current = segresource->Stat->Info.Size -
- (segresource->Stat->Info.HostNameOffset + __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1));
+ Assert(segresource->Stat->FailedTmpDirNum == 0);
+ segresource->Stat->Info.FailedTmpDirOffset = segresource->Stat->Info.HostNameOffset +
+ __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1);
if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
- current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
+ segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
- current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
+ segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
+ }
- /*
- * repalloc memory if new size exceeds the old one.
- * we don't shrink memory size if new size is less than the old one.
- */
- if (new > old && current < new)
- {
- SegStat newSegStat = rm_repalloc(PCONTEXT,
- segresource->Stat,
- offsetof(SegStatData, Info) +
- segresource->Stat->Info.Size + (new - old));
- segresource->Stat = newSegStat;
- segresource->Stat->Info.Size += (new - old);
- }
+ /* Clear old failed temporary directory string in SegInfoData */
+ memset((char *)&segresource->Stat->Info +
+ segresource->Stat->Info.FailedTmpDirOffset,
+ 0,
+ segresource->Stat->Info.Size -
+ segresource->Stat->Info.FailedTmpDirOffset);
- if (segresource->Stat->Info.FailedTmpDirOffset == 0)
- {
- Assert(segresource->Stat->FailedTmpDirNum == 0);
- segresource->Stat->Info.FailedTmpDirOffset = segresource->Stat->Info.HostNameOffset +
- __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1);
- if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
- segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
- if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
- segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
- }
+ if (segstat->FailedTmpDirNum != 0)
+ {
+ /* Update failed temporary directory string in SegInfoData */
+ memcpy((char *)&segresource->Stat->Info + segresource->Stat->Info.FailedTmpDirOffset,
+ GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
+ strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
+ }
+ else
+ {
+ segresource->Stat->Info.FailedTmpDirOffset = 0;
+ }
+ segresource->Stat->Info.FailedTmpDirLen = segstat->Info.FailedTmpDirLen;
+ segresource->Stat->FailedTmpDirNum = segstat->FailedTmpDirNum;
- /* clear old failed temporary directory string in SegInfoData */
- memset((char *)&segresource->Stat->Info +
- segresource->Stat->Info.FailedTmpDirOffset,
- 0,
- segresource->Stat->Info.Size -
- segresource->Stat->Info.FailedTmpDirOffset);
+ elog(LOG, "Resource manager change segment %s(%d) 's "
+ "failed temporary directory is changed to '%s'",
+ GET_SEGRESOURCE_HOSTNAME(segresource),
+ segid,
+ segresource->Stat->FailedTmpDirNum == 0 ?
+ "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
+
+ elog(RMLOG, "After resource manager "
+ "updates segment failed temporary directory, "
+ "GRM hostname:%s, GRM rackname:%s",
+ segresource->Stat->Info.GRMHostNameLen == 0 ?
+ "":GET_SEGINFO_GRMHOSTNAME(&(segresource->Stat->Info)),
+ segresource->Stat->Info.GRMRackNameLen == 0 ?
+ "":GET_SEGINFO_GRMRACKNAME(&(segresource->Stat->Info)));
+ }
- if (segstat->FailedTmpDirNum != 0)
- {
- memcpy((char *)&segresource->Stat->Info + segresource->Stat->Info.FailedTmpDirOffset,
- GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
- strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
- }
- else
- {
- segresource->Stat->Info.FailedTmpDirOffset = 0;
- }
- segresource->Stat->Info.FailedTmpDirLen = segstat->Info.FailedTmpDirLen;
- segresource->Stat->FailedTmpDirNum = segstat->FailedTmpDirNum;
-
- elog(RMLOG, "After resource manager "
- "updates segment failed temporary directory, "
- "GRM hostname:%s, GRM rackname:%s",
- segresource->Stat->Info.GRMHostNameLen == 0 ?
- "":GET_SEGINFO_GRMHOSTNAME(&(segresource->Stat->Info)),
- segresource->Stat->Info.GRMRackNameLen == 0 ?
- "":GET_SEGINFO_GRMRACKNAME(&(segresource->Stat->Info)));
-
- setSegResHAWQAvailability(segresource, segstat->FTSAvailable);
- if (Gp_role != GP_ROLE_UTILITY)
- {
- update_segment_failed_tmpdir(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
- segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
- SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
- segresource->Stat->FailedTmpDirNum,
- segresource->Stat->FailedTmpDirNum == 0 ?
- "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
- }
+ /* Set or clear failed temporary directory flag */
+ if (segresource->Stat->FailedTmpDirNum != 0)
+ {
+ segresource->Stat->StatusDesc |= SEG_STATUS_FAILED_TMPDIR;
+ }
+ else
+ {
+ if ((segresource->Stat->StatusDesc & SEG_STATUS_FAILED_TMPDIR) != 0)
+ segresource->Stat->StatusDesc &= ~SEG_STATUS_FAILED_TMPDIR;
+ }
+
+ if (segresource->Stat->StatusDesc == 0)
+ {
+ if (oldStatus == RESOURCE_SEG_STATUS_UNAVAILABLE ||
+ oldStatus == RESOURCE_SEG_STATUS_UNSET)
+ setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_AVAILABLE);
+ }
+ else
+ {
+ if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+ setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_UNAVAILABLE);
+ }
- if (statusChanged)
+ if (oldStatusDesc != segresource->Stat->StatusDesc ||
+ tmpDirChanged)
+ {
+ /*
+ * StatusDesc or failed temporary directory path is changed,
+ * update the gp_segment_configuration table and insert a row
+ * into gp_configuration_history table
+ */
+ if (Gp_role != GP_ROLE_UTILITY)
+ {
+ SimpStringPtr description = build_segment_status_description(segresource->Stat);
+ update_segment_status(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+ SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+
+ elog(LOG, "Resource manager update node(%s) information with heartbeat report,"
+ "status:'%c', description:%s",
+ GET_SEGRESOURCE_HOSTNAME(segresource),
+ IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+ SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+
+ add_segment_history_row(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(segresource),
+ IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+ SEG_STATUS_DESCRIPTION_UP:description->Str);
+ if (description != NULL)
{
- *capstatchanged = true;
- /*
- * Segment is set from up to down, return resource.
- */
- if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
- {
- returnAllGRMResourceFromSegment(segresource);
- }
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
}
+ }
+ }
- elog(LOG, "Master resource manager sets segment %s(%d)'s "
- "failed temporary directory to '%s', status:%c",
- GET_SEGRESOURCE_HOSTNAME(segresource),
- segid,
- segresource->Stat->FailedTmpDirNum == 0 ?
- "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
- segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
- SEGMENT_STATUS_UP : SEGMENT_STATUS_DOWN);
+ if (oldStatus != segresource->Stat->FTSAvailable)
+ {
+ *capstatchanged = true;
+ /*
+ * Segment is set from up to down, return resource.
+ */
+ if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+ {
+ returnAllGRMResourceFromSegment(segresource);
}
}
@@ -1161,6 +1323,7 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
SegResource segres = NULL;
SegStat newSegStat = NULL;
int32_t segid = SEGSTAT_ID_INVALID;
+ bool statusDescChange = false;
/* Anyway, the host GRM capacity is updated here if the cluster level
* capacity is fixed.
@@ -1305,10 +1468,8 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
elog(RMLOG, "After resource manager "
"updates segment info's GRM host name and rack name, "
"failed temporary directory: %s",
- segres->Stat->FailedTmpDirNum == 0 ? "":GET_SEGINFO_FAILEDTMPDIR(&(segres->Stat->Info)));
-
- /* Always set segment global resource manager available. */
- setSegResGLOBAvailability(segres, RESOURCE_SEG_STATUS_AVAILABLE);
+ segres->Stat->FailedTmpDirNum == 0 ?
+ "":GET_SEGINFO_FAILEDTMPDIR(&(segres->Stat->Info)));
if ( segres->Stat->GRMTotalMemoryMB != segstat->GRMTotalMemoryMB ||
segres->Stat->GRMTotalCore != segstat->GRMTotalCore )
@@ -1320,14 +1481,20 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
segres->Stat->GRMTotalMemoryMB = segstat->GRMTotalMemoryMB;
segres->Stat->GRMTotalCore = segstat->GRMTotalCore;
- /* Update overall cluster resource capacity. */
- minusResourceBundleData(&(PRESPOOL->GRMTotal), oldgrmmem, oldgrmcore);
- addResourceBundleData(&(PRESPOOL->GRMTotal),
- segres->Stat->GRMTotalMemoryMB,
- segres->Stat->GRMTotalCore);
+ /*
+ * If this segment is FTS available, then
+ * update overall cluster resource capacity.
+ */
+ if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
+ {
+ minusResourceBundleData(&(PRESPOOL->GRMTotal), oldgrmmem, oldgrmcore);
+ addResourceBundleData(&(PRESPOOL->GRMTotal),
+ segres->Stat->GRMTotalMemoryMB,
+ segres->Stat->GRMTotalCore);
+ }
elog(LOG, "Resource manager finds host %s capacity changed from "
- "GRM (%d MB, %d CORE) to GRM (%d MB, %d CORE)",
+ "GRM (%d MB, %d CORE) to GRM (%d MB, %d CORE)",
GET_SEGRESOURCE_HOSTNAME(segres),
oldgrmmem,
oldgrmcore,
@@ -1335,6 +1502,49 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
segres->Stat->GRMTotalCore);
}
+ segres->Stat->GRMHandled = true;
+ /* Clear no GRM node report flag. */
+ if ((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT) != 0)
+ {
+ segres->Stat->StatusDesc &= ~SEG_STATUS_NO_GRM_NODE_REPORT;
+ statusDescChange = true;
+ }
+
+ /*
+ * If get GRM node report, and there is no other flag,
+ * mark this segment to UP
+ */
+ if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) && segres->Stat->StatusDesc == 0)
+ {
+ Assert(statusDescChange == true);
+ setSegResHAWQAvailability(segres, RESOURCE_SEG_STATUS_AVAILABLE);
+ }
+
+ if (statusDescChange && Gp_role != GP_ROLE_UTILITY)
+ {
+ SimpStringPtr description = build_segment_status_description(segres->Stat);
+ update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+ SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+ SEG_STATUS_DESCRIPTION_UP:description->Str);
+
+ elog(LOG, "Resource manager update node(%s) information with yarn node report,"
+ "status:'%c', description:%s",
+ GET_SEGRESOURCE_HOSTNAME(segres),
+ IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+ SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+ (description->Len > 0)?description->Str:"");
+ if (description != NULL)
+ {
+ freeSimpleStringContent(description);
+ rm_pfree(PCONTEXT, description);
+ }
+ }
+
int32_t curratio = 0;
if (DRMGlobalInstance->ImpType == YARN_LIBYARN &&
segres->Stat->GRMTotalMemoryMB > 0 &&
@@ -1347,7 +1557,7 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
return FUNC_RETURN_OK;
}
-void setAllSegResourceGRMUnavailable(void)
+void setAllSegResourceGRMUnhandled(void)
{
List *allsegres = NULL;
ListCell *cell = NULL;
@@ -1356,7 +1566,7 @@ void setAllSegResourceGRMUnavailable(void)
foreach(cell, allsegres)
{
SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
- setSegResGLOBAvailability(segres, RESOURCE_SEG_STATUS_UNAVAILABLE);
+ segres->Stat->GRMHandled = false;
}
freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
}
@@ -1432,7 +1642,6 @@ SegResource createSegResource(SegStat segstat)
res->LastUpdateTime = gettime_microsec();
res->RUAlivePending = false;
res->Stat->FTSAvailable = RESOURCE_SEG_STATUS_UNSET;
- res->Stat->GRMAvailable = RESOURCE_SEG_STATUS_UNSET;
for ( int i = 0 ; i < RESOURCE_QUEUE_RATIO_SIZE ; ++i )
{
@@ -1455,13 +1664,6 @@ int setSegStatHAWQAvailability( SegStat segstat, uint8_t newstatus)
return res;
}
-int setSegStatGLOBAvailability( SegStat segstat, uint8_t newstatus)
-{
- int res = segstat->GRMAvailable;
- segstat->GRMAvailable = newstatus;
- return res;
-}
-
/* Set hawq status of host, return the old status */
int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
{
@@ -1499,12 +1701,7 @@ int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
segres->Stat->GRMTotalCore);
addNewResourceToResourceManagerByBundle(&(segres->Allocated));
- if ( (DRMGlobalInstance->ImpType == NONE_HAWQ2) ||
- (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
- IS_SEGSTAT_GRMAVAILABLE(segres->Stat)))
- {
- PRESPOOL->AvailNodeCount++;
- }
+ PRESPOOL->AvailNodeCount++;
}
else
{
@@ -1528,12 +1725,6 @@ int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
return res;
}
-int setSegResGLOBAvailability( SegResource segres, uint8_t newstatus)
-{
- int res = setSegStatGLOBAvailability(segres->Stat, newstatus);
- return res;
-}
-
/* Generate HAWQ host report. */
void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
{
@@ -1549,15 +1740,14 @@ void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
static char reporthead[256];
int headsize = sprintf(reporthead,
- "SEGMENT:ID=%d, "
- "HAWQAVAIL=%d,GLOBAVAIL=%d. "
- "FTS( %d MB, %d CORE). "
+ "SEGMENT:ID=%d, "
+ "HAWQAVAIL=%d. "
+ "FTS( %d MB, %d CORE). "
"GRM( %d MB, %d CORE). "
"MEM=%d(%d) MB. "
"CORE=%lf(%lf).\n",
seg->Stat->ID,
seg->Stat->FTSAvailable,
- seg->Stat->GRMAvailable,
seg->Stat->FTSTotalMemoryMB,
seg->Stat->FTSTotalCore,
seg->Stat->GRMTotalMemoryMB,
@@ -1565,7 +1755,7 @@ void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
seg->Allocated.MemoryMB,
seg->Available.MemoryMB,
seg->Allocated.Core,
- seg->Available.Core);
+ seg->Available.Core);
appendSelfMaintainBuffer(buff, reporthead, headsize);
generateSegInfoReport(&(seg->Stat->Info), buff);
}
@@ -1655,13 +1845,12 @@ void generateSegStatReport(SegStat segstat, SelfMaintainBuffer buff)
static char reporthead[256];
int reportheadlen = 0;
reportheadlen = sprintf(reporthead,
- "NODE:ID=%d,HAWQ %s, GRM %s, "
- "HAWQ CAP (%d MB, %lf CORE), "
+ "NODE:ID=%d,HAWQ %s, "
+ "HAWQ CAP (%d MB, %lf CORE), "
"GRM CAP(%d MB, %lf CORE),",
- segstat->ID,
+ segstat->ID,
segstat->FTSAvailable ? "AVAIL" : "UNAVAIL",
- segstat->GRMAvailable ? "AVAIL" : "UNAVAIL",
- segstat->FTSTotalMemoryMB,
+ segstat->FTSTotalMemoryMB,
segstat->FTSTotalCore * 1.0,
segstat->GRMTotalMemoryMB,
segstat->GRMTotalCore * 1.0);
@@ -3004,9 +3193,9 @@ void returnAllGRMResourceFromSegment(SegResource segres)
/*
* Go through each segment, and return the GRM containers in those segments
- * global resource manager unavailable.
+ * with GRM unavailable or FTS unavailable.
*/
-void returnAllGRMResourceFromGRMUnavailableSegments(void)
+void returnAllGRMResourceFromUnavailableSegments(void)
{
List *allsegres = NULL;
ListCell *cell = NULL;
@@ -3016,7 +3205,7 @@ void returnAllGRMResourceFromGRMUnavailableSegments(void)
foreach(cell, allsegres)
{
SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
- if ( IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+ if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
{
continue;
}
@@ -4208,8 +4397,7 @@ void refreshAvailableNodeCount(void)
SegResource segres = (SegResource)(pair->Value);
Assert( segres != NULL );
- if ( IS_SEGSTAT_FTSAVAILABLE(segres->Stat) &&
- IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+ if ( IS_SEGSTAT_FTSAVAILABLE(segres->Stat) )
{
PRESPOOL->AvailNodeCount++;
}
@@ -4335,10 +4523,12 @@ void fixClusterMemoryCoreRatio(void)
}
else
{
- if ( !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
- {
- continue;
- }
+ /*
+ * If this segment is FTS available,
+ * RM should get GRM report for it.
+ */
+ Assert((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT)
+ == 0);
memorymb = segres->Stat->GRMTotalMemoryMB;
core = segres->Stat->GRMTotalCore;
}
@@ -4389,10 +4579,12 @@ void fixClusterMemoryCoreRatio(void)
}
else
{
- if ( !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
- {
- continue;
- }
+ /*
+ * If this segment is FTS available,
+ * RM should get GRM report for it.
+ */
+ Assert((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT)
+ == 0);
memorymb = segres->Stat->GRMTotalMemoryMB;
core = segres->Stat->GRMTotalCore;
}
@@ -4563,8 +4755,7 @@ void adjustSegmentCapacityForGRM(SegResource segres)
adjustMemoryCoreValue(&(segres->Stat->GRMTotalMemoryMB),
&(segres->Stat->GRMTotalCore));
- if ( !IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
- !IS_SEGSTAT_GRMAVAILABLE(segres->Stat))
+ if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
{
return;
}
@@ -4628,9 +4819,8 @@ void dumpResourcePoolHosts(const char *filename)
segresource->Stat->FTSTotalCore,
segresource->Stat->GRMTotalMemoryMB,
segresource->Stat->GRMTotalCore);
- fprintf(fp, "HOST_AVAILABLITY(HAWQAvailable=%s:GLOBAvailable=%s)\n",
- segresource->Stat->FTSAvailable == 0 ? "false" : "true",
- segresource->Stat->GRMAvailable == 0 ? "false" : "true");
+ fprintf(fp, "HOST_AVAILABLITY(HAWQAvailable=%s)\n",
+ segresource->Stat->FTSAvailable == 0 ? "false" : "true");
fprintf(fp, "HOST_RESOURCE(AllocatedMemory=%d:AllocatedCores=%f:"
"AvailableMemory=%d:AvailableCores=%f:"
"IOBytesWorkload="INT64_FORMAT":"
@@ -4686,3 +4876,66 @@ void dumpResourcePoolHosts(const char *filename)
fclose(fp);
}
+
+static const char* SegStatusDesc[] = {
+ "heartbeat timeout",
+ "failed probing segment",
+ "communication error",
+ "failed temporary directory",
+ "resource manager process was reset",
+ "no global node report"
+};
+
+/*
+ * Build a string of status description for SegStat.
+ * This string contains the reason why this segment is DOWN.
+ * e.g. the failed temporary directories, heartbeat timeout.
+ */
+SimpStringPtr build_segment_status_description(SegStat segstat)
+{
+ SimpStringPtr description = createSimpleString(PCONTEXT);
+ SelfMaintainBufferData buf;
+ initializeSelfMaintainBuffer(&buf, PCONTEXT);
+ uint32_t tmp,cnt;
+
+ if (segstat->StatusDesc == 0)
+ goto _exit;
+
+ /* Count the number of flags */
+ cnt = tmp = segstat->StatusDesc;
+ while (tmp != 0)
+ {
+ tmp = tmp >> 1;
+ cnt -= tmp;
+ }
+
+ for (int idx = 0, tmp = 0; idx < sizeof(SegStatusDesc)/sizeof(char*); idx++)
+ {
+ if ((segstat->StatusDesc & 1<<idx) != 0)
+ {
+ tmp++;
+ appendSelfMaintainBuffer(&buf, SegStatusDesc[idx], strlen(SegStatusDesc[idx]));
+ if (1<<idx == SEG_STATUS_FAILED_TMPDIR)
+ {
+ appendSelfMaintainBuffer(&buf, ":", 1);
+ appendSelfMaintainBuffer(&buf,
+ GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
+ strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
+ }
+ if (tmp != cnt)
+ appendSelfMaintainBuffer(&buf, ";", 1);
+ }
+ }
+
+ if (buf.Size > 0)
+ {
+ static char zeropad = '\0';
+ appendSMBVar(&buf, zeropad);
+ setSimpleStringNoLen(description, buf.Buffer);
+ destroySelfMaintainBuffer(&buf);
+ }
+
+_exit:
+ return description;
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 00de72d..9af0d7c 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -2631,8 +2631,7 @@ void refreshActualMinGRMContainerPerSeg(void)
foreach(cell, allsegres)
{
SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
- if ( !IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
- !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+ if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
{
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/utils/gp/segadmin.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c
index 6b49926..775cadb 100644
--- a/src/backend/utils/gp/segadmin.c
+++ b/src/backend/utils/gp/segadmin.c
@@ -277,8 +277,7 @@ gp_add_master_standby(PG_FUNCTION_ARGS)
values[Anum_gp_segment_configuration_port - 1] = Int32GetDatum(master->port);
values[Anum_gp_segment_configuration_hostname - 1] = PG_GETARG_DATUM(0);
values[Anum_gp_segment_configuration_address - 1] = PG_GETARG_DATUM(1);
- nulls[Anum_gp_segment_configuration_failed_tmpdir_num - 1] = true;
- nulls[Anum_gp_segment_configuration_failed_tmpdir - 1] = true;
+ nulls[Anum_gp_segment_configuration_description - 1] = true;
tuple = caql_form_tuple(pcqCtx, values, nulls);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/gp_configuration.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/gp_configuration.h b/src/include/catalog/gp_configuration.h
index eaa0893..ee127e6 100644
--- a/src/include/catalog/gp_configuration.h
+++ b/src/include/catalog/gp_configuration.h
@@ -135,9 +135,10 @@ typedef FormData_gp_configuration *Form_gp_configuration;
CREATE TABLE gp_configuration_history
with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)
(
- time timestamp with time zone,
- dbid smallint,
- "desc" text
+ time timestamp with time zone,
+ registration_order integer,
+ hostname text,
+ description text
);
TIDYCAT_ENDDEF
@@ -146,8 +147,8 @@ typedef FormData_gp_configuration *Form_gp_configuration;
/* TIDYCAT_BEGIN_CODEGEN
WARNING: DO NOT MODIFY THE FOLLOWING SECTION:
- Generated by tidycat.pl version 21.
- on Wed Nov 24 14:13:16 2010
+ Generated by tidycat.pl version 34
+ on Fri Feb 26 10:43:15 2016
*/
@@ -182,9 +183,10 @@ typedef FormData_gp_configuration *Form_gp_configuration;
CATALOG(gp_configuration_history,5006) BKI_SHARED_RELATION BKI_WITHOUT_OIDS
{
- timestamptz time;
- int2 dbid;
- text desc;
+ timestamptz time;
+ int4 registration_order;
+ text hostname;
+ text description;
} FormData_gp_configuration_history;
#undef timestamptz
@@ -202,10 +204,11 @@ typedef FormData_gp_configuration_history *Form_gp_configuration_history;
* compiler constants for gp_configuration_history
* ----------------
*/
-#define Natts_gp_configuration_history 3
-#define Anum_gp_configuration_history_time 1
-#define Anum_gp_configuration_history_dbid 2
-#define Anum_gp_configuration_history_desc 3
+#define Natts_gp_configuration_history 4
+#define Anum_gp_configuration_history_time 1
+#define Anum_gp_configuration_history_registration_order 2
+#define Anum_gp_configuration_history_hostname 3
+#define Anum_gp_configuration_history_description 4
/* TIDYCAT_END_CODEGEN */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/gp_segment_config.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/gp_segment_config.h b/src/include/catalog/gp_segment_config.h
index f9f2def..fa577f1 100644
--- a/src/include/catalog/gp_segment_config.h
+++ b/src/include/catalog/gp_segment_config.h
@@ -53,8 +53,7 @@
port integer ,
hostname text ,
address text ,
- failed_tmpdir_num integer ,
- failed_tmpdir text
+ description text
);
create unique index on gp_segment_configuration(registration_order) with (indexid=6106, indexname=gp_segment_config_registration_order_index);
@@ -95,8 +94,7 @@ CATALOG(gp_segment_configuration,5036) BKI_SHARED_RELATION BKI_WITHOUT_OIDS
int4 port;
text hostname;
text address;
- int4 failed_tmpdir_num;
- text failed_tmpdir;
+ text description;
} FormData_gp_segment_configuration;
@@ -112,15 +110,14 @@ typedef FormData_gp_segment_configuration *Form_gp_segment_configuration;
* compiler constants for gp_segment_configuration
* ----------------
*/
-#define Natts_gp_segment_configuration 8
+#define Natts_gp_segment_configuration 7
#define Anum_gp_segment_configuration_registration_order 1
#define Anum_gp_segment_configuration_role 2
#define Anum_gp_segment_configuration_status 3
#define Anum_gp_segment_configuration_port 4
#define Anum_gp_segment_configuration_hostname 5
#define Anum_gp_segment_configuration_address 6
-#define Anum_gp_segment_configuration_failed_tmpdir_num 7
-#define Anum_gp_segment_configuration_failed_tmpdir 8
+#define Anum_gp_segment_configuration_description 7
/* TIDYCAT_END_CODEGEN */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/pg_proc.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index caf22a9..a6f8970 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -10196,7 +10196,6 @@ DESCR("Convert a cloned master catalog for use as a segment");
DATA(insert OID = 5053 ( gp_activate_standby PGNSP PGUID 12 f f f f v 0 16 f "" _null_ _null_ _null_ gp_activate_standby - _null_ n ));
DESCR("Activate a standby");
-
/* We cheat in the following two functions: they are technically volatile but */
/* we can only dispatch them if they're immutable :(. */
/* gp_add_segment_persistent_entries(int2, int2, _text) => bool */
@@ -10285,6 +10284,10 @@ DESCR("Remove an entry from gp_relation_node");
DATA(insert OID = 5074 ( gp_persistent_relation_node_check PGNSP PGUID 12 f f f t v 0 6990 f "" _null_ _null_ _null_ gp_persistent_relation_node_check - _null_ n ));
DESCR("physical filesystem information");
+/* gp_remove_segment_history() => bool */
+DATA(insert OID = 5075 ( gp_remove_segment_history PGNSP PGUID 12 f f f f v 0 16 f "" _null_ _null_ _null_ gp_remove_segment_history - _null_ n ));
+DESCR("Remove all entries from the gp_configuration_history");
+
/* cosh(float8) => float8 */
DATA(insert OID = 3539 ( cosh PGNSP PGUID 12 f f f f i 1 701 f "701" _null_ _null_ _null_ dcosh - _null_ n ));
DESCR("Hyperbolic cosine function");
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/utils/builtins.h
----------------------------------------------------------------------
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 48de877..55ff100 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -1120,6 +1120,7 @@ extern Datum gp_remove_master_standby(PG_FUNCTION_ARGS);
extern Datum gp_activate_standby(PG_FUNCTION_ARGS);
extern Datum gp_add_segment_mirror(PG_FUNCTION_ARGS);
+extern Datum gp_remove_segment_history(PG_FUNCTION_ARGS);
extern Datum gp_remove_segment_mirror(PG_FUNCTION_ARGS);
extern Datum gp_add_segment(PG_FUNCTION_ARGS);
extern Datum gp_remove_segment(PG_FUNCTION_ARGS);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/tools/bin/gppylib/data/2.0.json
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/data/2.0.json b/tools/bin/gppylib/data/2.0.json
index c65e12d..019c378 100644
--- a/tools/bin/gppylib/data/2.0.json
+++ b/tools/bin/gppylib/data/2.0.json
@@ -109,8 +109,9 @@
"CamelCaseRelationId" : "GpConfigHistoryRelationId",
"UppercaseReltypeOid" : "GP_CONFIGURATION_HISTORY_RELTYPE_OID",
"colh" : {
- "dbid" : "int2",
- "desc" : "text",
+ "description" : "text",
+ "hostname" : "text",
+ "registration_order" : "int4",
"time" : "timestamptz"
},
"cols" : [
@@ -121,19 +122,24 @@
"sqltype" : "timestamp_with_time_zone"
},
{
- "colname" : "dbid",
- "ctype" : "int2",
- "sqltype" : "smallint"
+ "colname" : "registration_order",
+ "ctype" : "int4",
+ "sqltype" : "integer"
+ },
+ {
+ "colname" : "hostname",
+ "ctype" : "text",
+ "sqltype" : "text"
},
{
- "colname" : "desc",
+ "colname" : "description",
"ctype" : "text",
"sqltype" : "text"
}
],
"filename" : "gp_configuration.h",
"relid_comment_tag" : "/* relation id: 5006 - gp_configuration_history */\n",
- "tabdef_text" : "\n CREATE TABLE gp_configuration_history\n with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)\n (\n time timestamp with time zone,\n dbid smallint,\n \"desc\" text\n )",
+ "tabdef_text" : "\n CREATE TABLE gp_configuration_history\n with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)\n (\n time timestamp with time zone,\n registration_order integer,\n hostname text,\n description text\n )",
"tzhack" : "\"time\"",
"with" : {
"bootstrap" : 0,
@@ -868,8 +874,7 @@
"UppercaseToastReltypeOid" : "GP_SEGMENT_CONFIGURATION_TOAST_RELTYPE_OID",
"colh" : {
"address" : "text",
- "failed_tmpdir" : "text",
- "failed_tmpdir_num" : "int4",
+ "description" : "text",
"hostname" : "text",
"port" : "int4",
"registration_order" : "int4",
@@ -909,12 +914,7 @@
"sqltype" : "text"
},
{
- "colname" : "failed_tmpdir_num",
- "ctype" : "int4",
- "sqltype" : "integer"
- },
- {
- "colname" : "failed_tmpdir",
+ "colname" : "description",
"ctype" : "text",
"sqltype" : "text"
}
@@ -955,7 +955,7 @@
}
],
"relid_comment_tag" : "/* relation id: 5036 - gp_segment_configuration */\n",
- "tabdef_text" : "\n CREATE TABLE gp_segment_configuration\n with (camelcase=GpSegmentConfig, shared=true, oid=false, relid=5036, reltype_oid=6442, toast_oid=2900, toast_index=2901, toast_reltype=2906, content=MASTER_ONLY)\n (\n registration_order integer ,\n role \"char\" ,\n status \"char\" ,\n port integer ,\n hostname text ,\n address text ,\n failed_tmpdir_num integer ,\n failed_tmpdir text\n )",
+ "tabdef_text" : "\n CREATE TABLE gp_segment_configuration\n with (camelcase=GpSegmentConfig, shared=true, oid=false, relid=5036, reltype_oid=6442, toast_oid=2900, toast_index=2901, toast_reltype=2906, content=MASTER_ONLY)\n (\n registration_order integer ,\n role \"char\" ,\n status \"char\" ,\n port integer ,\n hostname text ,\n address text ,\n description text\n )",
"with" : {
"bootstrap" : 0,
"camelcase" : "GpSegmentConfig",
[22/50] incubator-hawq git commit: Revert "HAWQ-533. Cursor failed,
if don't allocate resource in prepare."
Posted by sh...@apache.org.
Revert "HAWQ-533. Cursor failed, if don't allocate resource in prepare."
This reverts commit 96345af194bfd3ca05227037479c9835a5e495b0.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bc065a51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bc065a51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bc065a51
Branch: refs/heads/HAWQ-459
Commit: bc065a51fe1e04c15c398a6d15c5d01424efb52f
Parents: d259eff
Author: hzhang2 <zh...@163.com>
Authored: Thu Mar 17 09:18:50 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Thu Mar 17 09:18:50 2016 +0800
----------------------------------------------------------------------
src/backend/executor/spi.c | 24 +++++-------------------
1 file changed, 5 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bc065a51/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 9465de8..8e7645c 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1146,20 +1146,6 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
qtlist = copyObject(qtlist);
ptlist = copyObject(ptlist);
- PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
-
- if ( (Gp_role == GP_ROLE_DISPATCH) &&
- (stmt->resource_parameters != NULL) )
- {
- /*
- * Now, we want to allocate resource.
- */
- stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
- stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
- stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
- stmt->resource_parameters->vol_info_size);
- }
-
/* If the plan has parameters, set them up */
if (spiplan->nargs > 0)
{
@@ -1860,12 +1846,12 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
* We only allocate resource for multiple executions of queries, NOT for utility commands.
* SELECT/INSERT are supported at present.
*/
- if((queryTree->commandType == CMD_SELECT) ||
- (queryTree->commandType == CMD_INSERT))
+ if( (queryTree->commandType == CMD_SELECT) ||
+ (queryTree->commandType == CMD_INSERT) )
{
- if ((Gp_role == GP_ROLE_DISPATCH) &&
- (stmt->resource == NULL) &&
- (stmt->resource_parameters != NULL))
+ if ( (Gp_role == GP_ROLE_DISPATCH) &&
+ (stmt->resource == NULL) &&
+ (stmt->resource_parameters != NULL) )
{
SplitAllocResult *allocResult = NULL;
[33/50] incubator-hawq git commit: HAWQ-553. Analyze ROOTPATITION
using too few vsegs.
Posted by sh...@apache.org.
HAWQ-553. Analyze ROOTPATITION using too few vsegs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2513ad10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2513ad10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2513ad10
Branch: refs/heads/HAWQ-459
Commit: 2513ad108e0212b071da4301f19420c04c4db9d3
Parents: c29d647
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:31:58 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800
----------------------------------------------------------------------
src/backend/commands/analyze.c | 54 ++++++++++++++++++++++++-------------
1 file changed, 35 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2513ad10/src/backend/commands/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index dd5d5c0..32b3416 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -143,7 +143,7 @@ static void analyzeComputeAttributeStatistics(Oid relationOid,
float4 sampleTableRelTuples,
bool mergeStats,
AttributeStatistics *stats);
-static List* analyzableRelations(bool rootonly);
+static List* analyzableRelations(bool rootonly, List** fullRelOids);
static bool analyzePermitted(Oid relationOid);
static List *analyzableAttributes(Relation candidateRelation);
static int calculate_virtual_segment_number(List* candidateRelations);
@@ -300,6 +300,7 @@ void analyzeStatement(VacuumStmt *stmt, List *relids, int preferred_seg_num)
void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
{
List *lRelOids = NIL;
+ List *lFullRelOids = NIL;
MemoryContext callerContext = NULL;
MemoryContext analyzeStatementContext = NULL;
MemoryContext analyzeRelationContext = NULL;
@@ -379,7 +380,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
/**
* ANALYZE entire DB.
*/
- lRelOids = analyzableRelations(stmt->rootonly);
+ lRelOids = analyzableRelations(stmt->rootonly, &lFullRelOids);
if (stmt->rootonly && NIL == lRelOids)
{
ereport(WARNING,
@@ -392,6 +393,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
* ANALYZE called by autovacuum.
*/
lRelOids = relids;
+ lFullRelOids = relids;
}
else
{
@@ -419,11 +421,14 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
{
lRelOids = all_leaf_partition_relids(pn); /* all leaves */
}
+ lFullRelOids = all_leaf_partition_relids(pn);
lRelOids = lappend_oid(lRelOids, relationOid); /* root partition */
+ lFullRelOids = lappend_oid(lFullRelOids, relationOid); /* root partition */
if (optimizer_analyze_midlevel_partition)
{
lRelOids = list_concat(lRelOids, all_interior_partition_relids(pn)); /* interior partitions */
}
+ lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
}
else if (ps == PART_STATUS_INTERIOR) /* analyze an interior partition directly */
{
@@ -440,6 +445,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
else
{
lRelOids = list_make1_oid(relationOid);
+ lFullRelOids = list_make1_oid(relationOid);
}
}
@@ -472,6 +478,17 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
}
/**
+ * we use preferred_seg_num as default and
+ * compute target_seg_num based on data size and distributed type
+ * if there is no preferred_seg_num.
+ */
+ int target_seg_num = preferred_seg_num;
+ if (target_seg_num <= 0) {
+ target_seg_num = calculate_virtual_segment_number(lFullRelOids);
+ }
+ elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
+
+ /**
* We open relations with appropreciate locks
*/
List *candidateRelations = NIL;
@@ -499,17 +516,6 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
}
/**
- * we use preferred_seg_num as default and
- * compute target_seg_num based on data size and distributed type
- * if there is no preferred_seg_num.
- */
- int target_seg_num = preferred_seg_num;
- if (target_seg_num <= 0) {
- target_seg_num = calculate_virtual_segment_number(candidateRelations);
- }
- elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
-
- /**
* We allocate query resource for analyze
*/
QueryResource *resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
@@ -809,19 +815,23 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
* if there is hash distributed relations exist, use the max bucket number.
* if all relation are random, use the data size to determine vseg number.
*/
-static int calculate_virtual_segment_number(List* candidateRelations) {
+static int calculate_virtual_segment_number(List* candidateOids) {
ListCell* le1;
int vsegNumber = 1;
int64 totalDataSize = 0;
bool isHashRelationExist = false;
int maxHashBucketNumber = 0;
- foreach (le1, candidateRelations)
+ foreach (le1, candidateOids)
{
- Relation rel = (Relation)lfirst(le1);
- if (rel ) {
+ Oid candidateOid = InvalidOid;
+ candidateOid = lfirst_oid(le1);
+
+ //Relation rel = (Relation)lfirst(le1);
+ Relation rel = relation_open(candidateOid, AccessShareLock);
+ if (candidateOid > 0 ) {
GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
- rel->rd_id);
+ candidateOid);
if(targetPolicy == NULL){
return GetAnalyzeVSegNumLimit();
}
@@ -838,6 +848,7 @@ static int calculate_virtual_segment_number(List* candidateRelations) {
totalDataSize += calculate_relation_size(rel);
}
}
+ relation_close(rel, AccessShareLock);
}
if (isHashRelationExist) {
@@ -942,7 +953,7 @@ static bool analyzePermitted(Oid relationOid)
* Output:
* List of relids
*/
-static List* analyzableRelations(bool rootonly)
+static List* analyzableRelations(bool rootonly, List **fullRelOids)
{
List *lRelOids = NIL;
cqContext *pcqCtx;
@@ -957,6 +968,11 @@ static List* analyzableRelations(bool rootonly)
while (HeapTupleIsValid(tuple = caql_getnext(pcqCtx)))
{
Oid candidateOid = HeapTupleGetOid(tuple);
+ if (analyzePermitted(candidateOid)
+ && candidateOid != StatisticRelationId)
+ {
+ *fullRelOids = lappend_oid(*fullRelOids, candidateOid);
+ }
if (rootonly && !rel_is_partitioned(candidateOid))
{
continue;
[30/50] incubator-hawq git commit: HAWQ-552. long latency of
dispatching query resource due to wrong processing routine
Posted by sh...@apache.org.
HAWQ-552. long latency of dispatching query resource due to wrong processing routine
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3ae24a30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3ae24a30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3ae24a30
Branch: refs/heads/HAWQ-459
Commit: 3ae24a30b644c1acb939aab77931f410e6228aaf
Parents: 64663ef
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 12:40:37 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 12:40:37 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/resourcemanager.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3ae24a30/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 45a669e..21bf0d1 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -614,11 +614,8 @@ int MainHandlerLoop(void)
timeoutDeadResourceAllocation();
timeoutQueuedRequest();
- /* STEP 7. Generate output content to client connections. */
- sendResponseToClients();
-
/*
- * STEP 8. Check the status of all segment nodes, mark down if hasn't got
+ * STEP 7. Check the status of all segment nodes, mark down if hasn't got
* IMAlive message for a pre-defined period.
*/
uint64_t curtime = gettime_microsec();
@@ -631,7 +628,7 @@ int MainHandlerLoop(void)
PRESPOOL->LastCheckTime = curtime;
}
- /* STEP 9. Move all accepted GRM containers into resource pool. */
+ /* STEP 8. Move all accepted GRM containers into resource pool. */
moveAllAcceptedGRMContainersToResPool();
/*
@@ -657,7 +654,7 @@ int MainHandlerLoop(void)
setForcedReturnGRMContainerCount();
}
- /* STEP 10. Dispatch resource to queries and send the messages out.*/
+ /* STEP 9. Dispatch resource to queries and send the messages out.*/
if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 &&
PQUEMGR->toRunQueryDispatch &&
PQUEMGR->ForcedReturnGRMContainerCount == 0 &&
@@ -678,6 +675,9 @@ int MainHandlerLoop(void)
PQUEMGR->ForcedReturnGRMContainerCount);
}
+ /* STEP 10. Generate output content to client connections. */
+ sendResponseToClients();
+
/*
* STEP 11. Return containers to global resource manager if there some
* some idle resource.
[26/50] incubator-hawq git commit: Revert "Revert "HAWQ-547. When
HAWQ RM does not have valid YARN resource queue status received the cluster
report is not accepted""
Posted by sh...@apache.org.
Revert "Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted""
This reverts commit 2be5899ca55b954de90aa72d00c8bd02f8f2fb83.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1897ba1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1897ba1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1897ba1d
Branch: refs/heads/HAWQ-459
Commit: 1897ba1d00135f80a8d7ff656ca8389b2be213bd
Parents: 2be5899
Author: hubertzhang <hz...@pivotal.io>
Authored: Thu Mar 17 15:37:09 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Thu Mar 17 15:37:09 2016 +0800
----------------------------------------------------------------------
src/backend/resourcemanager/include/errorcode.h | 1 +
.../resourcebroker/resourcebroker_LIBYARN.c | 18 ++++++++++++++++--
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1897ba1d/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index ca338a8..de08fff 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,6 +162,7 @@ enum DRM_ERROR_CODE {
RESBROK_WRONG_GLOB_MGR_ADDRESS,
RESBROK_WRONG_GLOB_MGR_QUEUE,
RESBROK_WRONG_GLOB_MGR_APPNAME,
+ RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
RESBROK_PIPE_ERROR,
RESBROK_NOMORE_RESOURCE_IN_GRM,
RESBROK_WRONG_MESSAGE_ID,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1897ba1d/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index cdbe0d5..b8d205e 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
}
elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
- "maximum capacity is %lf.",
+ "maximum queue capacity is %lf.",
response.MachineCount,
response.QueueMaxCapacity);
@@ -631,6 +631,19 @@ int handleRB2RM_ClusterReport(void)
* TILL NOW, the whole message content is received.
*/
+ if ( res == FUNC_RETURN_OK )
+ {
+ /* Check if the YARN resource queue report is valid, i.e. maximum
+ * capacity and capacity are all greater than 0.
+ */
+ if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
+ {
+ elog(WARNING, "YARN mode resource broker got invalid cluster report");
+ res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
+
+ }
+ }
+
/* If something wrong, no need to keep the received content, free them. */
if ( res != FUNC_RETURN_OK )
{
@@ -1176,6 +1189,7 @@ void RB_LIBYARN_handleError(int errorcode)
}
else
{
- Assert(errorcode == FUNC_RETURN_OK);
+ Assert(errorcode == FUNC_RETURN_OK ||
+ errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
}
}
[46/50] incubator-hawq git commit: HAWQ-563. Restore guc
hawq_rm_nvseg_variance_amon_seg_respool_limit and fix its bug when resource
manager resets resource
Posted by sh...@apache.org.
HAWQ-563. Restore guc hawq_rm_nvseg_variance_amon_seg_respool_limit and fix its bug when resource manager resets resource
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/802d537c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/802d537c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/802d537c
Branch: refs/heads/HAWQ-459
Commit: 802d537cc8e19618fd2553608746ae595d8f8ec2
Parents: 0bb22dc
Author: YI JIN <yj...@pivotal.io>
Authored: Mon Mar 21 15:47:04 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Mon Mar 21 15:47:04 2016 +1100
----------------------------------------------------------------------
src/backend/cdb/cdbvars.c | 2 ++
.../resourcemanager/include/resourcepool.h | 1 +
src/backend/resourcemanager/resourcemanager.c | 1 +
src/backend/resourcemanager/resourcepool.c | 18 ++++++++++++++++--
src/backend/utils/misc/guc.c | 11 +++++++++++
5 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index a2476fd..b4a9f3d 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -390,6 +390,8 @@ double rm_regularize_io_factor;
double rm_regularize_usage_factor;
double rm_regularize_nvseg_factor;
+int rm_nvseg_variance_among_seg_respool_limit;
+
/* Greenplum Database Experimental Feature GUCs */
int gp_distinct_grouping_sets_threshold = 32;
bool gp_enable_explain_allstat = FALSE;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index f965c43..08ce305 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -633,6 +633,7 @@ int getOrderedResourceAllocTreeIndexByRatio(uint32_t ratio, BBST *tree);
void setAllSegResourceGRMUnhandled(void);
void resetAllSegmentsGRMContainerFailAllocCount(void);
+void resetAllSegmentsNVSeg(void);
struct RB_GRMContainerStatData
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index c585d40..c7215be 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -580,6 +580,7 @@ int MainHandlerLoop(void)
refreshMemoryCoreRatioLevelUsage(gettime_microsec());
resetAllSegmentsGRMContainerFailAllocCount();
+ resetAllSegmentsNVSeg();
/* Check if can resume using new available global resource manager.*/
if ( cleanedAllGRMContainers() )
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index acc8dbe..3a8d72b 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -1650,6 +1650,20 @@ void resetAllSegmentsGRMContainerFailAllocCount(void)
freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
}
+void resetAllSegmentsNVSeg(void)
+{
+ List *allsegres = NULL;
+ ListCell *cell = NULL;
+ getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres);
+
+ foreach(cell, allsegres)
+ {
+ SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
+ segres->NVSeg = 0;
+ }
+ freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
+}
+
/*
* Check index to get host id based on host name string.
*/
@@ -2556,7 +2570,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
Assert(minnvseg <= maxnvseg);
- if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_limit )
+ if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_respool_limit )
{
elog(LOG, "Reject virtual segment allocation based on data "
"locality information. After tentative allocation "
@@ -2565,7 +2579,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount,
"is %d, tolerated difference limit is %d.",
maxnvseg,
minnvseg,
- rm_nvseg_variance_among_seg_limit);
+ rm_nvseg_variance_among_seg_respool_limit);
/* Return the allocated resource. */
List *vsegcntlist = NULL;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 56909c9..a881b31 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6557,6 +6557,17 @@ static struct config_int ConfigureNamesInt[] =
},
{
+ {"hawq_rm_nvseg_variance_amon_seg_respool_limit", PGC_POSTMASTER, RESOURCES_MGM,
+ gettext_noop("the variance of vseg number in each segment that resource manager "
+ "should tolerate at most in resource pool when choosing segments "
+ "based on data locality reference."),
+ NULL
+ },
+ &rm_nvseg_variance_among_seg_respool_limit,
+ 2, 0, 65535, NULL, NULL
+ },
+
+ {
{"hawq_rm_container_batch_limit", PGC_POSTMASTER, RESOURCES_MGM,
gettext_noop("the batch process limit for global resource manager containers."),
NULL
[41/50] incubator-hawq git commit: Revert "HAWQ-532. Optimise vseg
number for copy to statement."
Posted by sh...@apache.org.
Revert "HAWQ-532. Optimise vseg number for copy to statement."
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/86d9b0c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/86d9b0c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/86d9b0c0
Branch: refs/heads/HAWQ-459
Commit: 86d9b0c03de349398ddb0337f7ac5dac3b58d2b9
Parents: 36571bc
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 14:22:29 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 14:22:29 2016 +0800
----------------------------------------------------------------------
src/backend/commands/copy.c | 97 ++++------------------------------------
1 file changed, 8 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/86d9b0c0/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e37fb83..bcd5384 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,11 +91,6 @@
#include "postmaster/autovacuum.h"
#include "cdb/dispatcher.h"
-/*
- * in dbsize.c
- */
-extern int64 calculate_relation_size(Relation rel);
-
/* DestReceiver for COPY (SELECT) TO */
typedef struct
{
@@ -142,7 +137,6 @@ static void copy_in_error_callback(void *arg);
static void CopyInitPartitioningState(EState *estate);
static void CopyInitDataParser(CopyState cstate);
static bool CopyCheckIsLastLine(CopyState cstate);
-static int calculate_virtual_segment_number(List* candidateRelations);
/* ==========================================================================
* The follwing macros aid in major refactoring of data processing code (in
@@ -1562,16 +1556,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
Assert(target_policy);
- /*
- * For hash table we use table bucket number to request vsegs
- * For random table, we use a fixed GUC value to request vsegs.
- */
- if(target_policy->nattrs > 0){
- target_segment_num = target_policy->bucketnum;
- }
- else{
- target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
- }
+ target_segment_num = target_policy->bucketnum;
pfree(target_policy);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1764,63 +1749,6 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
}
/*
- * calculate virtual segment number for copy statement.
- * if there is hash distributed relations exist, use the max bucket number.
- * if all relation are random, use the data size to determine vseg number.
- */
-static int calculate_virtual_segment_number(List* candidateOids) {
- ListCell* le1;
- int vsegNumber = 1;
- int64 totalDataSize = 0;
- bool isHashRelationExist = false;
- int maxHashBucketNumber = 0;
-
- foreach (le1, candidateOids)
- {
- Oid candidateOid = InvalidOid;
- candidateOid = lfirst_oid(le1);
-
- //Relation rel = (Relation)lfirst(le1);
- Relation rel = relation_open(candidateOid, AccessShareLock);
- if (candidateOid > 0 ) {
- GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
- candidateOid);
- if(targetPolicy == NULL){
- return GetAnalyzeVSegNumLimit();
- }
- if (targetPolicy->nattrs > 0) {
- isHashRelationExist = true;
- if(maxHashBucketNumber < targetPolicy->bucketnum){
- maxHashBucketNumber = targetPolicy->bucketnum;
- }
- }
- /*
- * if no hash relation, we calculate the data size of all the relations.
- */
- if (!isHashRelationExist) {
- totalDataSize += calculate_relation_size(rel);
- }
- }
- relation_close(rel, AccessShareLock);
- }
-
- if (isHashRelationExist) {
- vsegNumber = maxHashBucketNumber;
- } else {
- /*we allocate one virtual segment for each 128M data */
- totalDataSize >>= 27;
- vsegNumber = totalDataSize + 1;
- }
- Assert(vsegNumber > 0);
- /*vsegNumber should be less than GetUtilPartitionNum*/
- if(vsegNumber > GetQueryVsegNum()){
- vsegNumber = GetQueryVsegNum();
- }
-
- return vsegNumber;
-}
-
-/*
* This intermediate routine exists mainly to localize the effects of setjmp
* so we don't need to plaster a lot of variables with "volatile".
*/
@@ -1881,22 +1809,13 @@ DoCopyTo(CopyState cstate)
*/
if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
{
+ GpPolicy *target_policy = NULL;
int target_segment_num = 0;
- /*
- * copy hash table use table bucket number
- * copy random table use table size.
- */
- PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
- 0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
- List *lFullRelOids = NIL;
- if(pn){
- lFullRelOids = all_leaf_partition_relids(pn);
- lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
- }
- lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
- target_segment_num = calculate_virtual_segment_number(lFullRelOids);
- elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
+ target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
+ Assert(target_policy);
+ target_segment_num = target_policy->bucketnum;
+ pfree(target_policy);
cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
CopyToDispatch(cstate);
@@ -4281,7 +4200,7 @@ CopyFrom(CopyState cstate)
if (cstate->oids && file_has_oids)
MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
}
- else if (relstorage == RELSTORAGE_PARQUET)
+ else if ((relstorage == RELSTORAGE_PARQUET))
{
tuple = NULL;
}
@@ -4312,7 +4231,7 @@ CopyFrom(CopyState cstate)
{
HeapTuple newtuple;
- if(relstorage == RELSTORAGE_PARQUET)
+ if((relstorage == RELSTORAGE_PARQUET))
{
Assert(!tuple);
elog(ERROR, "triggers are not supported on tables that use column-oriented storage");
[09/50] incubator-hawq git commit: HAWQ-540. Change
output.replace-datanode-on-failure to true by default.
Posted by sh...@apache.org.
HAWQ-540. Change output.replace-datanode-on-failure to true by default.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a7ad3753
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a7ad3753
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a7ad3753
Branch: refs/heads/HAWQ-459
Commit: a7ad3753c064d6a7f8c4c28f691fc346d7d68c96
Parents: d178062
Author: zhenglin tao <zh...@gmail.com>
Authored: Tue Mar 15 10:12:36 2016 +0800
Committer: zhenglin tao <zh...@gmail.com>
Committed: Tue Mar 15 10:12:36 2016 +0800
----------------------------------------------------------------------
src/backend/utils/misc/etc/hdfs-client.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a7ad3753/src/backend/utils/misc/etc/hdfs-client.xml
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/etc/hdfs-client.xml b/src/backend/utils/misc/etc/hdfs-client.xml
index c0f395b..cf51e1a 100644
--- a/src/backend/utils/misc/etc/hdfs-client.xml
+++ b/src/backend/utils/misc/etc/hdfs-client.xml
@@ -236,7 +236,7 @@ HA -->
<!-- output client configuration -->
<property>
<name>output.replace-datanode-on-failure</name>
- <value>false</value>
+ <value>true</value>
<description>
whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
</description>
[50/50] incubator-hawq git commit: Merge remote-tracking branch
'origin/master' into HAWQ-459
Posted by sh...@apache.org.
Merge remote-tracking branch 'origin/master' into HAWQ-459
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/55be00ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/55be00ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/55be00ce
Branch: refs/heads/HAWQ-459
Commit: 55be00cee13cf1294ec75c7a50a928af735c2d4d
Parents: b1198ed 76dc631
Author: Shivram Mani <sh...@gmail.com>
Authored: Mon Mar 21 15:01:10 2016 -0700
Committer: Shivram Mani <sh...@gmail.com>
Committed: Mon Mar 21 15:01:10 2016 -0700
----------------------------------------------------------------------
configure | 27 -
configure.in | 27 -
.../hawq/mapreduce/TPCHClusterTester.java | 4 +-
src/backend/Makefile | 8 +-
src/backend/catalog/gp_toolkit.sql.in | 223 -----
src/backend/catalog/system_views.sql | 14 +-
src/backend/cdb/cdbcopy.c | 2 +-
src/backend/cdb/cdbdatalocality.c | 338 ++++---
src/backend/cdb/cdbdispatchresult.c | 28 +-
src/backend/cdb/cdbfilesplit.c | 67 +-
src/backend/cdb/cdbmutate.c | 2 +-
src/backend/cdb/cdbvars.c | 3 +
src/backend/cdb/dispatcher.c | 18 +-
src/backend/cdb/dispatcher_mgt.c | 9 +-
src/backend/cdb/executormgr.c | 32 +-
src/backend/cdb/motion/ic_common.c | 7 +-
src/backend/cdb/workermgr.c | 10 +
src/backend/commands/analyze.c | 54 +-
src/backend/commands/copy.c | 97 +-
src/backend/commands/explain.c | 2 +-
src/backend/commands/prepare.c | 2 +-
src/backend/commands/tablecmds.c | 50 +-
src/backend/executor/execDML.c | 99 +-
src/backend/executor/execMain.c | 13 +-
src/backend/executor/execUtils.c | 2 +-
src/backend/executor/nodeDML.c | 2 +-
src/backend/executor/spi.c | 71 +-
src/backend/gpopt/gpopt.mk | 18 +-
src/backend/gpopt/ivy.xml | 2 +-
.../gpopt/translate/CTranslatorDXLToPlStmt.cpp | 2 +
.../translate/CTranslatorRelcacheToDXL.cpp | 9 +
src/backend/gpopt/utils/COptTasks.cpp | 4 +-
src/backend/nodes/copyfuncs.c | 1 +
src/backend/nodes/outfast.c | 1 +
src/backend/nodes/outfuncs.c | 1 +
src/backend/nodes/readfast.c | 1 +
src/backend/optimizer/plan/planner.c | 60 +-
src/backend/optimizer/util/clauses.c | 11 +-
src/backend/optimizer/util/relnode.c | 2 +-
src/backend/optimizer/util/walkers.c | 1 +
src/backend/parser/analyze.c | 45 +-
src/backend/parser/gram.y | 9 +-
src/backend/postmaster/identity.c | 2 +-
src/backend/postmaster/postmaster.c | 2 -
.../communication/rmcomm_AsyncComm.c | 317 +++---
.../communication/rmcomm_Connect.c | 11 +-
.../communication/rmcomm_Message.c | 6 +-
.../communication/rmcomm_MessageHandler.c | 15 +-
.../communication/rmcomm_MessageServer.c | 134 +--
.../communication/rmcomm_QD2RM.c | 149 +--
.../communication/rmcomm_RM2RMSEG.c | 42 +-
.../communication/rmcomm_RMSEG2RM.c | 6 +-
.../communication/rmcomm_SyncComm.c | 108 +-
src/backend/resourcemanager/conntrack.c | 90 +-
.../include/communication/rmcomm_AsyncComm.h | 36 +-
.../include/communication/rmcomm_Connect.h | 1 -
.../communication/rmcomm_MessageServer.h | 12 +-
.../include/communication/rmcomm_SyncComm.h | 9 -
src/backend/resourcemanager/include/conntrack.h | 11 +-
src/backend/resourcemanager/include/errorcode.h | 1 +
.../resourcemanager/include/resourcepool.h | 47 +-
.../include/utils/network_utils.h | 40 +-
src/backend/resourcemanager/requesthandler.c | 178 ++--
.../resourcemanager/requesthandler_RMSEG.c | 6 +-
.../resourcebroker/resourcebroker_LIBYARN.c | 98 +-
.../resourcebroker_LIBYARN_proc.c | 3 +-
src/backend/resourcemanager/resourcemanager.c | 236 ++---
.../resourcemanager/resourcemanager_RMSEG.c | 1 -
src/backend/resourcemanager/resourcepool.c | 986 +++++++++++++------
src/backend/resourcemanager/resqueuemanager.c | 42 +-
.../resourcemanager/utils/network_utils.c | 407 +++++---
src/backend/tcop/postgres.c | 7 +-
src/backend/utils/adt/test/Makefile | 42 +
src/backend/utils/adt/test/varlena_test.c | 251 +++++
src/backend/utils/adt/varlena.c | 334 +++++--
src/backend/utils/cache/relcache.c | 2 +-
src/backend/utils/gp/segadmin.c | 3 +-
src/backend/utils/misc/etc/gpcheck.cnf | 2 +-
src/backend/utils/misc/etc/hdfs-client.xml | 2 +-
src/backend/utils/misc/faultinjector.c | 9 +-
src/backend/utils/misc/guc.c | 51 +-
src/bin/gpcheckhdfs/Makefile | 2 +-
src/bin/gpfilesystem/hdfs/Makefile | 2 +-
src/include/access/filesplit.h | 2 +-
src/include/catalog/gp_configuration.h | 27 +-
src/include/catalog/gp_segment_config.h | 11 +-
src/include/catalog/pg_proc.h | 5 +-
src/include/cdb/cdbdatalocality.h | 21 +-
src/include/cdb/cdbvars.h | 3 +
src/include/executor/execDML.h | 3 +-
src/include/executor/spi.h | 7 +
src/include/gpopt/translate/CTranslatorUtils.h | 4 -
src/include/nodes/execnodes.h | 7 +-
src/include/nodes/plannodes.h | 2 +
src/include/optimizer/planner.h | 4 +
src/include/postmaster/identity.h | 2 +-
src/include/utils/builtins.h | 1 +
src/include/utils/faultinjector.h | 2 +
src/include/utils/guc.h | 1 +
.../expected/create_table_distribution.out | 102 +-
src/test/regress/expected/function.out | 16 +
.../regress/expected/function_optimizer.out | 16 +
src/test/regress/expected/goh_partition.out | 126 +++
.../expected/gpsql_alter_table_optimizer.out | 2 +-
src/test/regress/expected/insert_optimizer.out | 2 +-
src/test/regress/output/exttab1.source | 6 +-
.../regress/output/exttab1_optimizer.source | 6 +-
.../regress/sql/create_table_distribution.sql | 41 +-
src/test/regress/sql/function.sql | 12 +
src/test/regress/sql/goh_partition.sql | 68 ++
tools/bin/gppylib/data/2.0.json | 32 +-
tools/bin/gppylib/programs/clsInjectFault.py | 5 +-
tools/bin/hawq_ctl | 117 ++-
tools/bin/hawqextract | 32 +-
tools/bin/hawqpylib/HAWQ_HELP.py | 2 +-
tools/bin/hawqpylib/hawqlib.py | 129 ++-
tools/bin/hawqpylib/programs/clsInjectFault.py | 5 +-
tools/bin/lib/hawqinit.sh | 66 +-
118 files changed, 3884 insertions(+), 2084 deletions(-)
----------------------------------------------------------------------
[17/50] incubator-hawq git commit: HAWQ-547. When HAWQ RM does not
have valid YARN resource queue status received the cluster report is not
accepted
Posted by sh...@apache.org.
HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5fda1826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5fda1826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5fda1826
Branch: refs/heads/HAWQ-459
Commit: 5fda1826aff4780b20e24db5210f79f867ef39ae
Parents: 8e38e84
Author: YI JIN <yj...@pivotal.io>
Authored: Wed Mar 16 13:52:37 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Wed Mar 16 13:52:37 2016 +1100
----------------------------------------------------------------------
src/backend/resourcemanager/include/errorcode.h | 1 +
.../resourcebroker/resourcebroker_LIBYARN.c | 18 ++++++++++++++++--
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5fda1826/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index ca338a8..de08fff 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,6 +162,7 @@ enum DRM_ERROR_CODE {
RESBROK_WRONG_GLOB_MGR_ADDRESS,
RESBROK_WRONG_GLOB_MGR_QUEUE,
RESBROK_WRONG_GLOB_MGR_APPNAME,
+ RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
RESBROK_PIPE_ERROR,
RESBROK_NOMORE_RESOURCE_IN_GRM,
RESBROK_WRONG_MESSAGE_ID,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5fda1826/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index cdbe0d5..b8d205e 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
}
elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
- "maximum capacity is %lf.",
+ "maximum queue capacity is %lf.",
response.MachineCount,
response.QueueMaxCapacity);
@@ -631,6 +631,19 @@ int handleRB2RM_ClusterReport(void)
* TILL NOW, the whole message content is received.
*/
+ if ( res == FUNC_RETURN_OK )
+ {
+ /* Check if the YARN resource queue report is valid, i.e. maximum
+ * capacity and capacity are all greater than 0.
+ */
+ if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
+ {
+ elog(WARNING, "YARN mode resource broker got invalid cluster report");
+ res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
+
+ }
+ }
+
/* If something wrong, no need to keep the received content, free them. */
if ( res != FUNC_RETURN_OK )
{
@@ -1176,6 +1189,7 @@ void RB_LIBYARN_handleError(int errorcode)
}
else
{
- Assert(errorcode == FUNC_RETURN_OK);
+ Assert(errorcode == FUNC_RETURN_OK ||
+ errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
}
}
[14/50] incubator-hawq git commit: HAWQ-499: Fix IPv6 on OSX > 10.11
Posted by sh...@apache.org.
HAWQ-499: Fix IPv6 on OSX > 10.11
- OSX IPv6 addresses are currently fetched with a command that
doesn't exist on OSX for versions < 10.11.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca2a7708
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca2a7708
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca2a7708
Branch: refs/heads/HAWQ-459
Commit: ca2a77088def9d22419cf0cf9db1fa01826e0cb0
Parents: c72d066
Author: Jacob Frank <jf...@pivotal.io>
Authored: Tue Mar 8 16:41:31 2016 -0800
Committer: rlei <rl...@pivotal.io>
Committed: Tue Mar 15 16:29:06 2016 +0800
----------------------------------------------------------------------
tools/bin/lib/hawqinit.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca2a7708/tools/bin/lib/hawqinit.sh
----------------------------------------------------------------------
diff --git a/tools/bin/lib/hawqinit.sh b/tools/bin/lib/hawqinit.sh
index af5635a..dab0b34 100755
--- a/tools/bin/lib/hawqinit.sh
+++ b/tools/bin/lib/hawqinit.sh
@@ -190,7 +190,7 @@ LOAD_GP_TOOLKIT () {
}
get_master_ipv6_addresses() {
- if [ "${distro_based_on}" = "Mac" ] && [ "${distro_version:0:5}" = "10.11" ]; then
+ if [ "${distro_based_on}" = "Mac" ]; then
MASTER_IPV6_LOCAL_ADDRESS_ALL=(`${IFCONFIG} | ${GREP} inet6 | ${AWK} '{print $2}' | cut -d'%' -f1`)
else
MASTER_IPV6_LOCAL_ADDRESS_ALL=(`ip -6 address show |${GREP} inet6|${AWK} '{print $2}' |cut -d'/' -f1`)
[06/50] incubator-hawq git commit: HAWQ-528. Reset
gp_connections_per_thread for dispatcher guc range from 1 to 512,
0 marks as invalid.
Posted by sh...@apache.org.
HAWQ-528. Reset gp_connections_per_thread for dispatcher guc range from 1 to 512, 0 marks as invalid.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4b53168b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4b53168b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4b53168b
Branch: refs/heads/HAWQ-459
Commit: 4b53168b537832f03aa6b4160a78afb63f34fbec
Parents: d5e7b92
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 17:39:31 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Tue Mar 15 09:38:40 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/dispatcher.c | 7 +------
src/backend/cdb/dispatcher_mgt.c | 9 +--------
src/backend/utils/misc/guc.c | 2 +-
3 files changed, 3 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 57ccdf4..20e97ab 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1656,12 +1656,7 @@ dispatcher_compute_threads_num(DispatchData *data)
/* TODO: decide the groups(threads) number. */
if (query_executors_num == 0)
threads_num = 1;
- if (executors_num_per_thread == 0)
- {
- threads_num = query_executors_num;
- executors_num_per_thread = 1;
- }
- else if (executors_num_per_thread > query_executors_num)
+ if (executors_num_per_thread > query_executors_num)
threads_num = 1;
else
threads_num = (query_executors_num / executors_num_per_thread) + ((query_executors_num % executors_num_per_thread == 0) ? 0 : 1);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher_mgt.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher_mgt.c b/src/backend/cdb/dispatcher_mgt.c
index 2920249..d0fd7e7 100644
--- a/src/backend/cdb/dispatcher_mgt.c
+++ b/src/backend/cdb/dispatcher_mgt.c
@@ -449,14 +449,7 @@ dispmgt_create_concurrent_connect_state(List *executors, int executors_num_per_t
int i, j;
/* Compute threads_num */
- if (executors_num == 0)
- return NIL;
- else if (executors_num_per_thread == 0)
- {
- threads_num = executors_num;
- executors_num_per_thread = 1;
- }
- else if (executors_num_per_thread > executors_num)
+ if (executors_num_per_thread > executors_num)
threads_num = 1;
else
threads_num = (executors_num / executors_num_per_thread) + ((executors_num % executors_num_per_thread == 0) ? 0 : 1);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5b04054..aac7192 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5647,7 +5647,7 @@ static struct config_int ConfigureNamesInt[] =
GUC_NOT_IN_SAMPLE | GUC_GPDB_ADDOPT
},
&gp_connections_per_thread,
- 512, 0, INT_MAX, assign_gp_connections_per_thread, show_gp_connections_per_thread
+ 512, 1, INT_MAX, assign_gp_connections_per_thread, show_gp_connections_per_thread
},
{
[45/50] incubator-hawq git commit: HAWQ-559. Fix QD hang when QE is
immediately killed after created
Posted by sh...@apache.org.
HAWQ-559. Fix QD hang when QE is immediately killed after created
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0bb22dce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0bb22dce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0bb22dce
Branch: refs/heads/HAWQ-459
Commit: 0bb22dce146cbd34507293b32f5603a6dc706aec
Parents: 6e88cba
Author: Lili Ma <ic...@gmail.com>
Authored: Fri Mar 18 16:49:06 2016 +0800
Committer: Lili Ma <ic...@gmail.com>
Committed: Mon Mar 21 09:49:06 2016 +0800
----------------------------------------------------------------------
src/backend/cdb/dispatcher.c | 2 ++
src/backend/cdb/executormgr.c | 20 +++++++++++++++-----
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 5d7105c..55a7823 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1240,6 +1240,8 @@ dispatch_run(DispatchData *data)
* Only after we have the executors, we can serialize the state. Or we
* don't know the executor listening address.
*/
+ CHECK_FOR_INTERRUPTS();
+
dispatcher_serialize_state(data);
dispatcher_serialize_query_resource(data);
dispatcher_set_state_run(data);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 47f5bbe..2bc5059 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -364,7 +364,19 @@ executormgr_validate_conn(PGconn *conn)
{
if (conn == NULL)
return false;
- return dispatch_validate_conn(conn->sock);
+ if (!dispatch_validate_conn(conn->sock))
+ {
+ printfPQExpBuffer(&conn->errorMessage,
+ libpq_gettext(
+ "server closed the connection unexpectedly\n"
+ "\tThis probably means the server terminated abnormally\n"
+ "\tbefore or while processing the request.\n"));
+ conn->status = CONNECTION_BAD;
+ closesocket(conn->sock);
+ conn->sock = -1;
+ return false;
+ }
+ return true;
}
/*
@@ -408,7 +420,7 @@ executormgr_dispatch_and_run(struct DispatchData *data, QueryExecutor *executor)
DispatchCommandQueryParms *parms = dispatcher_get_QueryParms(data);
if (!executormgr_is_dispatchable(executor))
- return false;
+ goto error;
TIMING_BEGIN(executor->time_dispatch_begin);
query = PQbuildGpQueryString(parms->strCommand, parms->strCommandlen,
@@ -573,10 +585,8 @@ executormgr_catch_error(QueryExecutor *executor)
msg = PQerrorMessage(conn);
- if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode == 0)){
+ if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode == 0)) {
errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
- if (errCode == ERRCODE_GP_INTERCONNECTION_ERROR)
- errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
}
PQExpBufferData selfDesc;
[21/50] incubator-hawq git commit: HAWQ-548. Fix wrong unit test case
for ip address parsing of interconnect
Posted by sh...@apache.org.
HAWQ-548. Fix wrong unit test case for ip address parsing of interconnect
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d259efff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d259efff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d259efff
Branch: refs/heads/HAWQ-459
Commit: d259efffbce3d9b1eb4e68646632d6419cf2c745
Parents: 1a75a2a
Author: Lili Ma <ic...@gmail.com>
Authored: Wed Mar 16 13:40:05 2016 +0800
Committer: Lili Ma <ic...@gmail.com>
Committed: Wed Mar 16 14:18:49 2016 +0800
----------------------------------------------------------------------
src/test/regress/expected/function.out | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d259efff/src/test/regress/expected/function.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/function.out b/src/test/regress/expected/function.out
index d493ed5..bda1ded 100644
--- a/src/test/regress/expected/function.out
+++ b/src/test/regress/expected/function.out
@@ -881,7 +881,6 @@ DROP FUNCTION outer(int);
DROP FUNCTION inner(int);
-- TEARDOWN
DROP TABLE foo;
-
-- HAWQ-510
drop table if exists testEntryDB;
NOTICE: table "testentrydb" does not exist, skipping
@@ -895,3 +894,6 @@ where t1.value=t2.value;
-----+-------
1 | 0
2 | 0
+(2 rows)
+
+drop table testEntryDB;
[38/50] incubator-hawq git commit: HAWQ-557. Set connection pool
connections always NODELAY and KEEPALIVE
Posted by sh...@apache.org.
HAWQ-557. Set connection pool connections always NODELAY and KEEPALIVE
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e82ac3f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e82ac3f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e82ac3f1
Branch: refs/heads/HAWQ-459
Commit: e82ac3f17319dcc630b467868491089a189aeeec
Parents: f38ad11
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:58:18 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:58:18 2016 +1100
----------------------------------------------------------------------
.../communication/rmcomm_AsyncComm.c | 6 +++++
.../communication/rmcomm_Connect.c | 5 ++++
.../communication/rmcomm_QD2RM.c | 27 ++++++++++++++++++++
.../include/utils/network_utils.h | 1 +
.../resourcemanager/utils/network_utils.c | 20 +++++++++++++++
5 files changed, 59 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c b/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
index 0084edc..181cc86 100644
--- a/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
+++ b/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
@@ -718,6 +718,12 @@ int registerAsyncConnectionFileDesc(const char *address,
sockres = connect(fd, (struct sockaddr *)&server_addr, sizeof(server_addr));
if ( sockres == 0 )
{
+ if ( setConnectionLongTermNoDelay(fd) != FUNC_RETURN_OK )
+ {
+ close(fd);
+ res = UTIL_NETWORK_FAIL_CONNECT;
+ goto exit;
+ }
/*
* New connection is created. Suppose domain socket and local socket
* connection can be done now. Register a normal client FD in poll() to
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_Connect.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_Connect.c b/src/backend/resourcemanager/communication/rmcomm_Connect.c
index fd916da..6c58a5c 100644
--- a/src/backend/resourcemanager/communication/rmcomm_Connect.c
+++ b/src/backend/resourcemanager/communication/rmcomm_Connect.c
@@ -68,6 +68,11 @@ void WriteReadyHandler_Connect(AsyncCommBuffer buffer)
shouldclose = true;
}
+ if ( setConnectionLongTermNoDelay(buffer->FD) != FUNC_RETURN_OK )
+ {
+ shouldclose = true;
+ }
+
if ( shouldclose )
{
ErrorHandler_Connect(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index d601383..59494a0 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -1440,6 +1440,7 @@ void *generateResourceRefreshHeartBeat(void *arg)
int sockres = 0;
while(true)
{
+ int on;
sockres = connect(fd,
(struct sockaddr *)&server_addr,
sizeof(server_addr));
@@ -1455,8 +1456,34 @@ void *generateResourceRefreshHeartBeat(void *arg)
"failed to connect to resource manager, "
"fd %d (errno %d)", fd, errno);
close(fd);
+ fd = -1;
}
}
+#ifdef TCP_NODELAY
+ on = 1;
+ if (sockres == 0 &&
+ setsockopt(fd,
+ IPPROTO_TCP, TCP_NODELAY,
+ (char *) &on, sizeof(on)) < 0)
+ {
+ write_log("ERROR setsockopt(TCP_NODELAY) failed: %m");
+ close(fd);
+ fd = -1;
+ sockres = -1;
+ }
+#endif
+ on = 1;
+ if (sockres == 0 &&
+ setsockopt(fd,
+ SOL_SOCKET, SO_KEEPALIVE,
+ (char *) &on, sizeof(on)) < 0)
+ {
+ write_log("ERROR setsockopt(SO_KEEPALIVE) failed: %m");
+ close(fd);
+ fd = -1;
+ sockres = -1;
+ }
+
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/include/utils/network_utils.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/utils/network_utils.h b/src/backend/resourcemanager/include/utils/network_utils.h
index 4aa2cf0..a048131 100644
--- a/src/backend/resourcemanager/include/utils/network_utils.h
+++ b/src/backend/resourcemanager/include/utils/network_utils.h
@@ -109,6 +109,7 @@ int setConnectionNonBlocked(int fd);
#define DRM_SOCKET_CONN_RETRY 5
int connectToServerRemote(const char *address,uint16_t port,int *clientfd);
+int setConnectionLongTermNoDelay(int fd);
void closeConnectionRemote(int *clientfd);
void returnAliveConnectionRemote(int *clientfd,
const char *hostname,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/utils/network_utils.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/utils/network_utils.c b/src/backend/resourcemanager/utils/network_utils.c
index b8bbab9..fbfe0a9 100644
--- a/src/backend/resourcemanager/utils/network_utils.c
+++ b/src/backend/resourcemanager/utils/network_utils.c
@@ -715,3 +715,23 @@ static void cleanupSocketConnectionPool(int code, Datum arg)
freePAIRRefList(&ResolvedHostnames, &addrlist);
cleanHASHTABLE(&ResolvedHostnames);
}
+
+int setConnectionLongTermNoDelay(int fd)
+{
+ int on;
+#ifdef TCP_NODELAY
+ on = 1;
+ if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &on, sizeof(on)) < 0)
+ {
+ elog(WARNING, "setsockopt(TCP_NODELAY) failed: %m");
+ return SYSTEM_CALL_ERROR;
+ }
+#endif
+ on = 1;
+ if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (char *) &on, sizeof(on)) < 0)
+ {
+ elog(WARNING, "setsockopt(SO_KEEPALIVE) failed: %m");
+ return SYSTEM_CALL_ERROR;
+ }
+ return FUNC_RETURN_OK;
+}