You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by sh...@apache.org on 2016/03/21 23:01:30 UTC

[01/50] incubator-hawq git commit: HAWQ-527. Remove dead code of generateResourceRefreshHeartBeat()

Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-459 b1198ed9c -> 55be00cee


HAWQ-527. Remove dead code of generateResourceRefreshHeartBeat()


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b3e2add6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b3e2add6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b3e2add6

Branch: refs/heads/HAWQ-459
Commit: b3e2add6d346ecde3c03307f27fcdcf006c1fa70
Parents: fa993ac
Author: YI JIN <yj...@pivotal.io>
Authored: Mon Mar 14 16:48:55 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Mon Mar 14 16:48:55 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/communication/rmcomm_QD2RM.c | 6 ------
 1 file changed, 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b3e2add6/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index a645dd4..d601383 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -1519,12 +1519,6 @@ void *generateResourceRefreshHeartBeat(void *arg)
 		pg_usleep(rm_session_lease_heartbeat_interval * 1000000L);
 	}
 
-	if ( fd >= 0 )
-	{
-		close(fd);
-		fd = -1;
-	}
-
 	freeHeartBeatThreadArg(&tharg);
 	write_log("generateResourceRefreshHeartBeat exits.");
 	return 0;


[19/50] incubator-hawq git commit: HAWQ-531. Change GUC default value.

Posted by sh...@apache.org.
HAWQ-531. Change GUC default value.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d1a8540
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d1a8540
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d1a8540

Branch: refs/heads/HAWQ-459
Commit: 4d1a85405b87145fb931a16be9e22bd40f16eca8
Parents: 96345af
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 11:08:57 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 11:08:57 2016 +0800

----------------------------------------------------------------------
 src/backend/utils/misc/guc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d1a8540/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fa58f86..ba8e747 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6379,7 +6379,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_limit,
-            1000, 1, 65535, NULL, NULL
+            512, 1, 65535, NULL, NULL
     },
 
     {
@@ -6535,7 +6535,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&rm_rejectrequest_nseg_limit,
-		2, 0, 65535, NULL, NULL
+		4, 0, 65535, NULL, NULL
 	},
 
 	{


[25/50] incubator-hawq git commit: Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted"

Posted by sh...@apache.org.
Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted"

This reverts commit 5fda1826aff4780b20e24db5210f79f867ef39ae.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2be5899c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2be5899c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2be5899c

Branch: refs/heads/HAWQ-459
Commit: 2be5899ca55b954de90aa72d00c8bd02f8f2fb83
Parents: aef7bd2
Author: YI JIN <yj...@pivotal.io>
Authored: Thu Mar 17 16:41:40 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Thu Mar 17 16:41:40 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/include/errorcode.h   |  1 -
 .../resourcebroker/resourcebroker_LIBYARN.c       | 18 ++----------------
 2 files changed, 2 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2be5899c/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index de08fff..ca338a8 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,7 +162,6 @@ enum DRM_ERROR_CODE {
 	RESBROK_WRONG_GLOB_MGR_ADDRESS,
 	RESBROK_WRONG_GLOB_MGR_QUEUE,
 	RESBROK_WRONG_GLOB_MGR_APPNAME,
-	RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
 	RESBROK_PIPE_ERROR,
 	RESBROK_NOMORE_RESOURCE_IN_GRM,
 	RESBROK_WRONG_MESSAGE_ID,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2be5899c/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index b8d205e..cdbe0d5 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
 	}
 
 	elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
-			  "maximum queue capacity is %lf.",
+			  "maximum capacity is %lf.",
 			  response.MachineCount,
 			  response.QueueMaxCapacity);
 
@@ -631,19 +631,6 @@ int handleRB2RM_ClusterReport(void)
 	 * TILL NOW, the whole message content is received.
 	 */
 
-	if ( res == FUNC_RETURN_OK )
-	{
-		/* Check if the YARN resource queue report is valid, i.e. maximum
-		 * capacity and capacity are all greater than 0.
-		 */
-		if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
-		{
-			elog(WARNING, "YARN mode resource broker got invalid cluster report");
-			res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
-
-		}
-	}
-
 	/* If something wrong, no need to keep the received content, free them. */
 	if ( res != FUNC_RETURN_OK )
 	{
@@ -1189,7 +1176,6 @@ void RB_LIBYARN_handleError(int errorcode)
 	}
 	else
 	{
-		Assert(errorcode == FUNC_RETURN_OK ||
-			   errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
+		Assert(errorcode == FUNC_RETURN_OK);
 	}
 }


[35/50] incubator-hawq git commit: HAWQ-531. Change GUC default value.

Posted by sh...@apache.org.
HAWQ-531. Change GUC default value.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/eaaf9451
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/eaaf9451
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/eaaf9451

Branch: refs/heads/HAWQ-459
Commit: eaaf94514396c8f60716a78a76852017f7614808
Parents: 4d997b5
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:16:51 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800

----------------------------------------------------------------------
 src/backend/utils/misc/guc.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/eaaf9451/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 3d13eb4..a6a8bee 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4516,7 +4516,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&default_hash_table_bucket_number,
-		6, 1, INT_MAX, NULL, NULL
+		6, 1, 10000, NULL, NULL
 	},
 
 	{
@@ -4525,7 +4525,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_copy_from_perquery,
-		6, 1, INT_MAX, NULL, NULL
+		6, 1, 10000, NULL, NULL
 	},
 
 	{
@@ -4534,7 +4534,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_analyze_perquery_perseg_limit,
-		4, 1, INT_MAX, NULL, NULL
+		4, 1, 10000, NULL, NULL
 	},
 
 	{
@@ -4543,7 +4543,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_analyze_perquery_limit,
-		256, 1, INT_MAX, NULL, NULL
+		256, 1, 10000, NULL, NULL
 	},
 
 	{
@@ -6388,7 +6388,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_limit,
-            1000, 1, 65535, NULL, NULL
+            512, 1, 10000, NULL, NULL
     },
 
     {
@@ -6398,7 +6398,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_perseg_limit,
-            6, 1, 65535, NULL, NULL
+            6, 1, 10000, NULL, NULL
     },
 
     {


[28/50] incubator-hawq git commit: HAWQ-551. Add datalocility time to explain analyse result

Posted by sh...@apache.org.
HAWQ-551. Add datalocility time to explain analyse result


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/dfc67313
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/dfc67313
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/dfc67313

Branch: refs/heads/HAWQ-459
Commit: dfc673135f9f1c92487423436428be82ad1db1e4
Parents: 94b7a82
Author: ivan <iw...@pivotal.io>
Authored: Thu Mar 17 15:07:27 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Fri Mar 18 09:32:31 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c    | 28 +++++++++++++++++++++++++---
 src/backend/commands/explain.c       |  2 +-
 src/backend/optimizer/plan/planner.c |  1 +
 src/include/cdb/cdbdatalocality.h    |  1 +
 src/include/nodes/plannodes.h        |  1 +
 5 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index c1f829b..eec87b4 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -309,6 +309,10 @@ typedef struct split_to_segment_mapping_context {
 	int64 total_file_count;
 
 	int64 total_metadata_logic_len;
+
+    int metadata_cache_time_us;
+    int alloc_resource_time_us;
+    int cal_datalocality_time_us;
 } split_to_segment_mapping_context;
 
 typedef struct vseg_list{
@@ -569,6 +573,10 @@ static void init_datalocality_context(split_to_segment_mapping_context *context)
 	context->total_split_count = 0;
 	context->total_file_count = 0;
 	context->total_metadata_logic_len = 0;
+
+    context->metadata_cache_time_us = 0;
+    context->alloc_resource_time_us = 0;
+    context->cal_datalocality_time_us = 0;
 	return;
 }
 
@@ -928,6 +936,8 @@ int64 get_block_locations_and_claculte_table_size(split_to_segment_mapping_conte
 	}
 	context->total_file_count = totalFileCount;
 	context->total_size = total_size;
+    
+    context->metadata_cache_time_us = eclaspeTime;
 
 	if(debug_datalocality_time){
 		elog(LOG, "metadata overall execution time: %d us. \n", eclaspeTime);
@@ -3395,13 +3405,13 @@ static void print_datalocality_overall_log_information(SplitAllocResult *result,
 	appendStringInfo(result->datalocalityInfo, "data locality ratio: %.3f; virtual segment number: %d; "
 			"different host number: %d; virtual segment number per host(avg/min/max): (%d/%d/%d); "
 			"segment size(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); "
-			"segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f)."
+			"segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f); "
 			,log_context->datalocalityRatio,assignment_context->virtual_segment_num,log_context->numofDifferentHost,
 			log_context->avgSegmentNumofHost,log_context->minSegmentNumofHost,log_context->maxSegmentNumofHost,
 			log_context->avgSizeOverall,log_context->minSizeSegmentOverall,log_context->maxSizeSegmentOverall,
 			log_context->avgSizeOverallPenalty,log_context->minSizeSegmentOverallPenalty,log_context->maxSizeSegmentOverallPenalty,
-			log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall
-			);
+			log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall);
+
 	if (debug_fake_datalocality) {
 			fprintf(fp, "datalocality ratio: %.3f; virtual segments number: %d, "
 					"different host number: %d, segment number per host(avg/min/max): (%d/%d/%d); "
@@ -3919,9 +3929,17 @@ run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments, Query
 	uint64_t run_datalocality = 0;
 	run_datalocality = gettime_microsec();
 	int dl_overall_time = run_datalocality - before_run_allocation;
+    
+    context->cal_datalocality_time_us = dl_overall_time; 
+
 	if(debug_datalocality_time){
 		elog(LOG, "datalocality overall execution time: %d us. \n", dl_overall_time);
 	}
+
+    result->datalocalityTime = (double)(context->metadata_cache_time_us + context->alloc_resource_time_us + context->cal_datalocality_time_us)/ 1000;
+    appendStringInfo(result->datalocalityInfo, "DFS metadatacache: %.3f ms; resource allocation: %.3f ms; datalocality calculation: %.3f ms.",
+            (double)context->metadata_cache_time_us/1000, (double)context->alloc_resource_time_us/1000, (double)context->cal_datalocality_time_us/1000);  
+
 	return alloc_result;
 }
 
@@ -3980,6 +3998,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	result->relsType = NIL;
 	result->planner_segments = -1;
 	result->datalocalityInfo = makeStringInfo();
+    result->datalocalityTime = 0;
 
 	/* fake data locality */
 	if (debug_fake_datalocality) {
@@ -4230,6 +4249,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			}
 			uint64_t after_rm_allocate_resource = gettime_microsec();
 			int eclaspeTime = after_rm_allocate_resource - before_rm_allocate_resource;
+        
+            context.alloc_resource_time_us = eclaspeTime;
+
 			if(debug_datalocality_time){
 				elog(LOG, "rm allocate resource overall execution time: %d us. \n", eclaspeTime);
 			}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/commands/explain.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 51afde2..0e7bff5 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -470,7 +470,7 @@ ExplainOnePlan_internal(PlannedStmt *plannedstmt,
     MemoryContext   oldcxt = CurrentMemoryContext;
 	QueryDesc  *queryDesc;
 	instr_time	starttime;
-	double		totaltime = 0;
+	double		totaltime = plannedstmt->datalocalityTime / 1000;
     StringInfo  buf = &es->outbuf;
     EState     *estate = NULL;
 	int			eflags;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index be057e0..6b2e13e 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -459,6 +459,7 @@ planner(Query *parse, int cursorOptions,
 		result->scantable_splits = ppResult->saResult.alloc_results;
 		result->planner_segments = ppResult->saResult.planner_segments;
 		result->datalocalityInfo = ppResult->saResult.datalocalityInfo;
+        result->datalocalityTime = ppResult->saResult.datalocalityTime;
 	}
 	plannerLevel--;
 	if ((ppResult != NULL))

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 95f2bd5..d28fc3e 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -46,6 +46,7 @@ typedef struct SplitAllocResult
   int planner_segments;
   List *relsType;// relation type after datalocality changing
   StringInfo datalocalityInfo;
+  double datalocalityTime;
 } SplitAllocResult;
 
 /*

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/nodes/plannodes.h
----------------------------------------------------------------------
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index ebccb9d..7c10cd5 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -204,6 +204,7 @@ typedef struct PlannedStmt
 		MemoryAccount *memoryAccount;
 
 		StringInfo datalocalityInfo;
+        double     datalocalityTime;
 } PlannedStmt;
 
 


[49/50] incubator-hawq git commit: HAWQ-566. Fix analyze and truncate hung.

Posted by sh...@apache.org.
HAWQ-566. Fix analyze and truncate hung.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/76dc6319
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/76dc6319
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/76dc6319

Branch: refs/heads/HAWQ-459
Commit: 76dc63191fb21b1321c00f00954afc1cb98c6f45
Parents: 7daee40
Author: hzhang2 <zh...@163.com>
Authored: Mon Mar 21 16:38:15 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:38:15 2016 +0800

----------------------------------------------------------------------
 src/backend/commands/analyze.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/76dc6319/src/backend/commands/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 32b3416..ac12837 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -828,7 +828,7 @@ static int calculate_virtual_segment_number(List* candidateOids) {
 		candidateOid = lfirst_oid(le1);
 
 		//Relation rel = (Relation)lfirst(le1);
-		Relation rel = relation_open(candidateOid, AccessShareLock);
+		Relation rel = relation_open(candidateOid, ShareUpdateExclusiveLock);
 		if (candidateOid > 0 ) {
 			GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
 					candidateOid);
@@ -848,7 +848,7 @@ static int calculate_virtual_segment_number(List* candidateOids) {
 				totalDataSize += calculate_relation_size(rel);
 			}
 		}
-		relation_close(rel, AccessShareLock);
+		relation_close(rel, NoLock);
 	}
 
 	if (isHashRelationExist) {


[40/50] incubator-hawq git commit: HAWQ-532. Optimise vseg number for copy to statement.

Posted by sh...@apache.org.
HAWQ-532. Optimise vseg number for copy to statement.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/36571bce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/36571bce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/36571bce

Branch: refs/heads/HAWQ-459
Commit: 36571bce2c96fce04ceb94f9ab78ace4cc90c66d
Parents: b3a3521
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 12:36:39 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 12:36:39 2016 +0800

----------------------------------------------------------------------
 src/backend/commands/copy.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/36571bce/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 2f4cf39..e37fb83 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -1888,11 +1888,12 @@ DoCopyTo(CopyState cstate)
 			 */
 			PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
 													0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
-			Assert(pn);
 			List		*lFullRelOids = NIL;
-			lFullRelOids = all_leaf_partition_relids(pn);
-			lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id); /* root partition */
-			lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+			if(pn){
+				lFullRelOids = all_leaf_partition_relids(pn);
+				lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+			}
+			lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
 
 			target_segment_num = calculate_virtual_segment_number(lFullRelOids);
 			elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);


[48/50] incubator-hawq git commit: HAWQ-562. Refactor bucket number of external table.

Posted by sh...@apache.org.
HAWQ-562. Refactor bucket number of external table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7daee400
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7daee400
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7daee400

Branch: refs/heads/HAWQ-459
Commit: 7daee4002780797587c3dcca265c646fd0c92e1e
Parents: 695606f
Author: hzhang2 <zh...@163.com>
Authored: Mon Mar 21 10:26:11 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:29:42 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbcopy.c         |  2 +-
 src/backend/cdb/cdbdatalocality.c | 97 ++++++++++++++++++++--------------
 src/backend/cdb/cdbfilesplit.c    | 67 +++++++----------------
 src/backend/commands/tablecmds.c  | 11 ++--
 src/backend/parser/analyze.c      |  2 +-
 src/include/access/filesplit.h    |  2 +-
 src/include/cdb/cdbdatalocality.h |  1 -
 7 files changed, 88 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbcopy.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbcopy.c b/src/backend/cdb/cdbcopy.c
index 17ffa96..ee2ff80 100644
--- a/src/backend/cdb/cdbcopy.c
+++ b/src/backend/cdb/cdbcopy.c
@@ -222,7 +222,7 @@ cdbCopyStart(CdbCopy *c, char *copyCmd, Oid relid, Oid relerror, List *err_aoseg
 		List *scantable_splits = NIL;
 		prepareDispatchedCatalogRelation(q->contextdisp, relid, FALSE, NULL);
 		scantable_splits = AssignAOSegFileSplitToSegment(relid, NIL,
-														true, c->partition_num,
+														c->partition_num,
 														scantable_splits);
 		((CopyStmt *)q->utilityStmt)->scantable_splits = scantable_splits;
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index 3f8fcb5..4c67429 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -34,6 +34,7 @@
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
 #include "catalog/catquery.h"
+#include "catalog/pg_exttable.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
@@ -296,11 +297,13 @@ typedef struct split_to_segment_mapping_context {
 	int64 split_size;
 	MemoryContext old_memorycontext;
 	MemoryContext datalocality_memorycontext;
-	int externTableSegNum;  //expected virtual segment number when external table exists
+	int externTableOnClauseSegNum;  //expected virtual segment number when external table exists
+	int externTableLocationSegNum;  //expected virtual segment number when external table exists
 	int tableFuncSegNum;  //expected virtual segment number when table function exists
 	int hashSegNum;  // expected virtual segment number when there is hash table in from clause
 	int randomSegNum; // expected virtual segment number when there is random table in from clause
 	int resultRelationHashSegNum; // expected virtual segment number when hash table as a result relation
+	int minimum_segment_num; //default is 1.
 	int64 randomRelSize; //all the random relation size
 	int64 hashRelSize; //all the hash relation size
 
@@ -529,13 +532,15 @@ static void init_datalocality_context(split_to_segment_mapping_context *context)
 	context->rtc_context.range_tables = NIL;
 	context->rtc_context.full_range_tables = NIL;
 
-	context->externTableSegNum = 0;
+	context->externTableOnClauseSegNum = 0;
+	context->externTableLocationSegNum = 0;
 	context->tableFuncSegNum = 0;
 	context->hashSegNum = 0;
 	context->resultRelationHashSegNum = 0;
 	context->randomSegNum = 0;
 	context->randomRelSize = 0;
 	context->hashRelSize = 0;
+	context->minimum_segment_num = 1;
 	/*
 	 * initialize the data distribution
 	 * static context.
@@ -762,17 +767,24 @@ static void check_keep_hash_and_external_table(
 			/* targetPolicy->bucketnum is bucket number of external table,
 			 * whose default value is set to default_segment_num
 			 */
-			if (context->externTableSegNum == 0) {
-				context->externTableSegNum = targetPolicy->bucketnum;
-			} else {
-				if (context->externTableSegNum < targetPolicy->bucketnum) {
-					context->externTableSegNum = targetPolicy->bucketnum;
-					/*
-					 * In this case, two external table join but with different bucket number
-					 * we cannot allocate the right segment number.
-					 */
-					//now we just restrict that vseg num > bucket number of external table
-					//elog(ERROR, "All external tables in one query must have the same bucket number!");
+			ExtTableEntry* extEnrty = GetExtTableEntry(rel->rd_id);
+			if(extEnrty->isweb){
+				if (context->externTableOnClauseSegNum == 0) {
+					context->externTableOnClauseSegNum = targetPolicy->bucketnum;
+				} else {
+					if (context->externTableOnClauseSegNum != targetPolicy->bucketnum) {
+						/*
+						 * In this case, two external table join but with different bucket number
+						 * we cannot allocate the right segment number.
+						 */
+						elog(ERROR, "All external tables in one query must have the same bucket number!");
+					}
+				}
+			}
+			else{
+				if (context->externTableLocationSegNum < targetPolicy->bucketnum) {
+					context->externTableLocationSegNum = targetPolicy->bucketnum;
+					context->minimum_segment_num =  targetPolicy->bucketnum;
 				}
 			}
 		}
@@ -4074,17 +4086,17 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				&(context.rtc_context.range_tables), &isTableFunctionExists,
 				context.datalocality_memorycontext);
 
-		/*Table Function VSeg Number = default_segment_number(configured in GUC) if table function exists,
+		/* set expected virtual segment number for hash table and external table*/
+		/* calculate hashSegNum, externTableSegNum, resultRelationHashSegNum */
+		check_keep_hash_and_external_table(&context, query, intoPolicy);
+
+		/*Table Function VSeg Number = default_segment_number(configured in GUC) if table function exists or gpfdist exists,
 		 *0 Otherwise.
 		 */
 		if (isTableFunctionExists) {
 			context.tableFuncSegNum = GetUserDefinedFunctionVsegNum();
 		}
 
-		/* set expected virtual segment number for hash table and external table*/
-		/* calculate hashSegNum, externTableSegNum, resultRelationHashSegNum */
-		check_keep_hash_and_external_table(&context, query, intoPolicy);
-
 		/* get block location and calculate relation size*/
 		get_block_locations_and_claculte_table_size(&context);
 
@@ -4163,13 +4175,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				context.randomSegNum = expected_segment_num_with_max_filecount;
 			}
 			/* Step4 we at least use one segment*/
-			if (context.randomSegNum < minimum_segment_num) {
-				context.randomSegNum = minimum_segment_num;
+			if (context.randomSegNum < context.minimum_segment_num) {
+				context.randomSegNum = context.minimum_segment_num;
 			}
 
 			int maxExpectedNonRandomSegNum = 0;
-			if (maxExpectedNonRandomSegNum < context.externTableSegNum)
-				maxExpectedNonRandomSegNum = context.externTableSegNum;
 			if (maxExpectedNonRandomSegNum < context.tableFuncSegNum)
 				maxExpectedNonRandomSegNum = context.tableFuncSegNum;
 			if (maxExpectedNonRandomSegNum < context.hashSegNum)
@@ -4183,7 +4193,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				fprintf(fpsegnum, "Result relation hash segment num : %d.\n", context.resultRelationHashSegNum);
 				fprintf(fpsegnum, "\n");
 				fprintf(fpsegnum, "Table  function      segment num : %d.\n", context.tableFuncSegNum);
-				fprintf(fpsegnum, "Extern table         segment num : %d.\n", context.externTableSegNum);
+				fprintf(fpsegnum, "Extern table         segment num : %d.\n", context.externTableOnClauseSegNum);
 				fprintf(fpsegnum, "From hash relation   segment num : %d.\n", context.hashSegNum);
 				fprintf(fpsegnum, "MaxExpectedNonRandom segment num : %d.\n", maxExpectedNonRandomSegNum);
 				fprintf(fpsegnum, "\n");
@@ -4193,29 +4203,33 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			int maxTargetSegmentNumber = 0;
 			/* we keep resultRelationHashSegNum in the highest priority*/
 			if (context.resultRelationHashSegNum != 0) {
-				if (context.resultRelationHashSegNum < context.externTableSegNum
-						&& context.externTableSegNum != 0) {
+				if ((context.resultRelationHashSegNum != context.externTableOnClauseSegNum
+						&& context.externTableOnClauseSegNum != 0)
+						|| (context.resultRelationHashSegNum < context.externTableLocationSegNum)) {
 					cleanup_allocation_algorithm(&context);
 					elog(ERROR, "Could not allocate enough memory! "
 							"bucket number of result hash table and external table should match each other");
 				}
 				maxTargetSegmentNumber = context.resultRelationHashSegNum;
 				minTargetSegmentNumber = context.resultRelationHashSegNum;
-			} else if (maxExpectedNonRandomSegNum > 0) {
+			}
+			else if(context.externTableOnClauseSegNum > 0){
 				/* bucket number of external table must be the same with the number of virtual segments*/
-				if (maxExpectedNonRandomSegNum == context.externTableSegNum) {
-					context.externTableSegNum =
-							context.externTableSegNum < minimum_segment_num ?
-									minimum_segment_num : context.externTableSegNum;
-					maxTargetSegmentNumber = context.externTableSegNum;
-					minTargetSegmentNumber = context.externTableSegNum;
-				} else if (maxExpectedNonRandomSegNum == context.hashSegNum) {
+				if(context.externTableOnClauseSegNum < context.externTableLocationSegNum){
+					cleanup_allocation_algorithm(&context);
+					elog(ERROR, "external table bucket number should match each other");
+				}
+				maxTargetSegmentNumber = context.externTableOnClauseSegNum;
+				minTargetSegmentNumber = context.externTableOnClauseSegNum;
+			}
+			else if (maxExpectedNonRandomSegNum > 0) {
+				if (maxExpectedNonRandomSegNum == context.hashSegNum) {
 					/* in general, we keep bucket number of hash table equals to the number of virtual segments
 					 * but this rule can be broken when there is a large random table in the range tables list
 					 */
 					context.hashSegNum =
-							context.hashSegNum < minimum_segment_num ?
-							minimum_segment_num : context.hashSegNum;
+							context.hashSegNum < context.minimum_segment_num ?
+							context.minimum_segment_num : context.hashSegNum;
 					double considerRandomWhenHashExistRatio = 1.5;
 					/*if size of random table >1.5 *hash table, we consider relax the restriction of hash bucket number*/
 					if (context.randomRelSize
@@ -4224,7 +4238,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 							context.randomSegNum = context.hashSegNum;
 						}
 						maxTargetSegmentNumber = context.randomSegNum;
-						minTargetSegmentNumber = minimum_segment_num;
+						minTargetSegmentNumber = context.minimum_segment_num;
 					} else {
 						maxTargetSegmentNumber = context.hashSegNum;
 						minTargetSegmentNumber = context.hashSegNum;
@@ -4232,17 +4246,17 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				} else if (maxExpectedNonRandomSegNum == context.tableFuncSegNum) {
 					/* if there is a table function, we should at least use tableFuncSegNum virtual segments*/
 					context.tableFuncSegNum =
-							context.tableFuncSegNum < minimum_segment_num ?
-									minimum_segment_num : context.tableFuncSegNum;
+							context.tableFuncSegNum < context.minimum_segment_num ?
+									context.minimum_segment_num : context.tableFuncSegNum;
 					if (context.randomSegNum < context.tableFuncSegNum) {
 						context.randomSegNum = context.tableFuncSegNum;
 					}
 					maxTargetSegmentNumber = context.randomSegNum;
-					minTargetSegmentNumber = minimum_segment_num;
+					minTargetSegmentNumber = context.minimum_segment_num;
 				}
 			} else {
 				maxTargetSegmentNumber = context.randomSegNum;
-				minTargetSegmentNumber = minimum_segment_num;
+				minTargetSegmentNumber = context.minimum_segment_num;
 			}
 
 			if (enforce_virtual_segment_number > 0) {
@@ -4254,6 +4268,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				maxTargetSegmentNumber = fixedVsegNum;
 				minTargetSegmentNumber = fixedVsegNum;
 			}
+			if(maxTargetSegmentNumber < minTargetSegmentNumber){
+				maxTargetSegmentNumber = minTargetSegmentNumber;
+			}
 			uint64_t before_rm_allocate_resource = gettime_microsec();
 
 			/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/cdb/cdbfilesplit.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbfilesplit.c b/src/backend/cdb/cdbfilesplit.c
index 4a2ad04..128f949 100644
--- a/src/backend/cdb/cdbfilesplit.c
+++ b/src/backend/cdb/cdbfilesplit.c
@@ -44,8 +44,7 @@ static void fileSplit_free(FileSplitNode *split);
 
 static List *
 computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits,
-						List *segment_infos, bool keep_hash_policy,
-						int target_segment_num);
+						List *segment_infos, int target_segment_num);
 
 static List *
 postProcessSplitsPerSegment(List *oldSplitToSegmentMaps);
@@ -54,7 +53,7 @@ static int
 fileSplitNodeCmp(const void *left, const void *right);
 
 static SegFileSplitMapNode *
-AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num)
+AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num)
 {
 	SegFileSplitMapNode *result = NULL;
 	char storageChar;
@@ -108,7 +107,7 @@ AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_ha
 	 * data locality into account.
 	 */
 	splitToSegmentMaps = computeSplitToSegmentMaps(relid, targetPolicy, splits, segment_infos,
-											keep_hash_policy, target_segment_num);
+											target_segment_num);
 	Assert(splitToSegmentMaps);
 	list_free(splits);
 
@@ -118,7 +117,7 @@ AssignSingleAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_ha
 }
 
 List *
-AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num, List *existings)
+AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num, List *existings)
 {
 	SegFileSplitMapNode *result = NULL;
 
@@ -130,7 +129,7 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
 		foreach (child, children)
 		{
 			Oid myrelid = lfirst_oid(child);
-			result = AssignSingleAOSegFileSplitToSegment(myrelid, segment_infos, keep_hash_policy,
+			result = AssignSingleAOSegFileSplitToSegment(myrelid, segment_infos,
 					target_segment_num);
 			existings = lappend(existings, result);
 		}
@@ -140,7 +139,7 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
 	}
 	else
 	{
-		result = AssignSingleAOSegFileSplitToSegment(relid, segment_infos, keep_hash_policy,
+		result = AssignSingleAOSegFileSplitToSegment(relid, segment_infos,
 													target_segment_num);
 		existings = lappend(existings, result);
 	}
@@ -149,25 +148,14 @@ AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_pol
 }
 
 /*
- * If keep_hash_policy is false, then the table is treated as randomly
- * distributed, regardless of its distribution policy.
  *
  * If segment_infos is NIL, then data locality is not needed.
  */
 static List *
 computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits, List *segment_infos,
-						bool keep_hash_policy, int target_segment_num)
+						int target_segment_num)
 {
 	List *splitToSegmentMaps = NIL;
-	if (keep_hash_policy)
-	{
-		/*
-		 * If we want to keep the hash distribution policy,
-		 * we need to make sure the data partition number
-		 * equals to the number of target segments.
-		 */
-		Assert(targetPolicy->bucketnum == target_segment_num);
-	}
 
 	if (segment_infos != NIL)
 	{
@@ -184,37 +172,22 @@ computeSplitToSegmentMaps(Oid relid, GpPolicy *targetPolicy, List *splits, List
 	 */
 	if (segment_infos == NIL)
 	{
-		/*
-		 * Sub case 1: we need to keep the hash distribution policy.
-		 *
-		 * In this case, we just need consider the segment file number.
-		 * This case works exactly the same as HAWQ1.x.
-		 */
-		if (keep_hash_policy)
+		ListCell *lc;
+		int i;
+		for (i = 0; i < target_segment_num; i++)
 		{
-			ListCell *lc;
-			int i;
-			for (i = 0; i < target_segment_num; i++)
-			{
-				splitToSegmentMaps = lappend(splitToSegmentMaps, NIL);
-			}
-
-			foreach(lc, splits)
-			{
-				int assigned_seg_no;
-				ListCell *per_seg_split;
-				FileSplit split = (FileSplitNode *)lfirst(lc);
-				Assert(split);
-				assigned_seg_no = (split->segno - 1) % target_segment_num;
-				per_seg_split = list_nth_cell(splitToSegmentMaps, assigned_seg_no);
-				lfirst(per_seg_split) = lappend((List *)lfirst(per_seg_split), split);
-			}
+			splitToSegmentMaps = lappend(splitToSegmentMaps, NIL);
 		}
-		else
+
+		foreach(lc, splits)
 		{
-			ereport(ERROR,
-					(errcode(ERRCODE_CDB_FEATURE_NOT_YET),
-					errmsg("Assigning splits to segment without keeping hash distribution policy is not allowed")));
+			int assigned_seg_no;
+			ListCell *per_seg_split;
+			FileSplit split = (FileSplitNode *)lfirst(lc);
+			Assert(split);
+			assigned_seg_no = (split->segno - 1) % target_segment_num;
+			per_seg_split = list_nth_cell(splitToSegmentMaps, assigned_seg_no);
+			lfirst(per_seg_split) = lappend((List *)lfirst(per_seg_split), split);
 		}
 	}
 	else

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/commands/tablecmds.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index c47b828..259821b 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -968,7 +968,12 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt)
 			locationUris = transformLocationUris(exttypeDesc->location_list,
 												 createExtStmt->formatOpts,
 												 isweb, iswritable);
-			
+			int locLength = list_length(exttypeDesc->location_list);
+			if (createStmt->policy && locLength > 0)
+			{
+				createStmt->policy->bucketnum = locLength;
+			}
+
 			break;
 
 		case EXTTBL_TYPE_EXECUTE:
@@ -6437,7 +6442,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap)
 										 tab->relid,
 										 false,
 										 NULL);
-		ar_tab->scantable_splits = AssignAOSegFileSplitToSegment(tab->relid, NIL, true,
+		ar_tab->scantable_splits = AssignAOSegFileSplitToSegment(tab->relid, NIL,
 		                    target_segment_num, ar_tab->scantable_splits);
 		/*
 		 * Specify the segno directly as we don't have segno mapping here.
@@ -16719,7 +16724,7 @@ ATPExecPartSplit(Relation rel,
      * Dispatch split-related metadata.
      */
     scantable_splits = AssignAOSegFileSplitToSegment((Oid)intVal((Value *)pc->partid),
-              NIL, true, target_segment_num, scantable_splits);
+              NIL, target_segment_num, scantable_splits);
 
     pc->scantable_splits = scantable_splits;
     pc->newpart_aosegnos = segment_segnos;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/backend/parser/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c
index f17c90a..1f48c36 100644
--- a/src/backend/parser/analyze.c
+++ b/src/backend/parser/analyze.c
@@ -2724,7 +2724,7 @@ transformETDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
 										 sizeof(p->attrs[0]));
 			p->ptype = POLICYTYPE_PARTITIONED;
 			p->nattrs = 0;
-			p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
+			p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetExternalTablePartitionNum());
 			p->attrs[0] = 1;
 		}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/include/access/filesplit.h
----------------------------------------------------------------------
diff --git a/src/include/access/filesplit.h b/src/include/access/filesplit.h
index e075b18..e627b07 100644
--- a/src/include/access/filesplit.h
+++ b/src/include/access/filesplit.h
@@ -65,7 +65,7 @@ typedef SegFileSplitMapNode *SegFileSplitMap;
  * table data.
  */
 List *
-AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, bool keep_hash_policy, int target_segment_num, List *existings);
+AssignAOSegFileSplitToSegment(Oid relid, List *segment_infos, int target_segment_num, List *existings);
 
 /*
  * Given the relid, and the segment index, return the splits assigned

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7daee400/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 224cb53..21e4c68 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -33,7 +33,6 @@
 #include "nodes/parsenodes.h"
 #include "executor/execdesc.h"
 
-#define minimum_segment_num 1
 /*
  * structure containing information about data residence
  * at the host.


[18/50] incubator-hawq git commit: HAWQ-533. Cursor failed, if don't allocate resource in prepare.

Posted by sh...@apache.org.
HAWQ-533. Cursor failed, if don't allocate resource in prepare.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/96345af1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/96345af1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/96345af1

Branch: refs/heads/HAWQ-459
Commit: 96345af194bfd3ca05227037479c9835a5e495b0
Parents: 5fda182
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 10:59:38 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 11:02:25 2016 +0800

----------------------------------------------------------------------
 src/backend/executor/spi.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96345af1/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 8e7645c..9465de8 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1146,6 +1146,20 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
 	qtlist = copyObject(qtlist);
 	ptlist = copyObject(ptlist);
 
+	PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
+
+	if ( (Gp_role == GP_ROLE_DISPATCH) &&
+			 (stmt->resource_parameters != NULL) )
+	{
+		/*
+		 * Now, we want to allocate resource.
+		 */
+		stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
+				stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
+				stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
+				stmt->resource_parameters->vol_info_size);
+	}
+
 	/* If the plan has parameters, set them up */
 	if (spiplan->nargs > 0)
 	{
@@ -1846,12 +1860,12 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 				 * We only allocate resource for multiple executions of queries, NOT for utility commands.
 				 * SELECT/INSERT are supported at present.
 				 */
-				if( (queryTree->commandType == CMD_SELECT) ||
-				    (queryTree->commandType == CMD_INSERT) )
+				if((queryTree->commandType == CMD_SELECT) ||
+						(queryTree->commandType == CMD_INSERT))
 				{
-					if ( (Gp_role == GP_ROLE_DISPATCH) &&
-					     (stmt->resource == NULL) &&
-					     (stmt->resource_parameters != NULL) )
+					if ((Gp_role == GP_ROLE_DISPATCH) &&
+							(stmt->resource == NULL) &&
+							(stmt->resource_parameters != NULL))
 					{
 						SplitAllocResult *allocResult = NULL;
 


[31/50] incubator-hawq git commit: HAWQ-555. Remove unused guc for allocating resource for tolerating variance of number of virtual segments

Posted by sh...@apache.org.
HAWQ-555. Remove unused guc for allocating resource for tolerating variance of number of virtual segments


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2c033d9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2c033d9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2c033d9f

Branch: refs/heads/HAWQ-459
Commit: 2c033d9f069bb2fe092dba3ec4ed9cb2d2e13e5f
Parents: 3ae24a3
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:41:49 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:41:49 2016 +1100

----------------------------------------------------------------------
 src/backend/cdb/cdbvars.c                  |  2 --
 src/backend/resourcemanager/resourcepool.c |  4 ++--
 src/backend/utils/misc/guc.c               | 11 -----------
 3 files changed, 2 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index b4a9f3d..a2476fd 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -390,8 +390,6 @@ double	rm_regularize_io_factor;
 double	rm_regularize_usage_factor;
 double	rm_regularize_nvseg_factor;
 
-int		rm_nvseg_variance_among_seg_respool_limit;
-
 /* Greenplum Database Experimental Feature GUCs */
 int         gp_distinct_grouping_sets_threshold = 32;
 bool		gp_enable_explain_allstat = FALSE;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index ca275f5..e5d083c 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2556,7 +2556,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 		freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
 		Assert(minnvseg <= maxnvseg);
 
-		if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_respool_limit )
+		if ( maxnvseg - minnvseg > rm_tolerate_nseg_limit )
 		{
 			elog(LOG, "Reject virtual segment allocation based on data "
 					  "locality information. After tentative allocation "
@@ -2565,7 +2565,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 					  "is %d, tolerated difference limit is %d.",
 					  maxnvseg,
 					  minnvseg,
-					  rm_nvseg_variance_among_seg_respool_limit);
+					  rm_tolerate_nseg_limit);
 
 			/* Return the allocated resource. */
 			List 	 *vsegcntlist = NULL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2c033d9f/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 54ef13f..3d13eb4 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6557,17 +6557,6 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
-		{"hawq_rm_nvseg_variance_amon_seg_respool_limit", PGC_POSTMASTER, RESOURCES_MGM,
-			gettext_noop("the variance of vseg number in each segment that resource manager "
-						 "should tolerate at most in resource pool when choosing segments "
-						 "based on data locality reference."),
-			NULL
-		},
-		&rm_nvseg_variance_among_seg_respool_limit,
-		2, 0, 65535, NULL, NULL
-	},
-
-	{
 		{"hawq_rm_container_batch_limit", PGC_POSTMASTER, RESOURCES_MGM,
 			gettext_noop("the batch process limit for global resource manager containers."),
 			NULL


[44/50] incubator-hawq git commit: HAWQ-502 drop boost-thread link flag [#105338218]

Posted by sh...@apache.org.
HAWQ-502 drop boost-thread link flag [#105338218]

Because it's unnecessary.
This fixes #420


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6e88cbab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6e88cbab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6e88cbab

Branch: refs/heads/HAWQ-459
Commit: 6e88cbabfe985bbb3f38f32a821ae84158735419
Parents: c67a1a4
Author: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Authored: Fri Mar 18 17:01:38 2016 -0700
Committer: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <xz...@pivotal.io>
Committed: Fri Mar 18 17:03:40 2016 -0700

----------------------------------------------------------------------
 src/backend/Makefile               | 2 +-
 src/bin/gpcheckhdfs/Makefile       | 2 +-
 src/bin/gpfilesystem/hdfs/Makefile | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 18ed91a..f89c24d 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,7 +52,7 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
 
 # The backend doesn't need everything that's in LIBS, however
 LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
 
 # adding orca libraries
 ifeq ($(enable_orca),yes)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index cd81902..fd3c4f1 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
 
 OBJS=gpcheckhdfs.o 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6e88cbab/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index b0fb359..cfe1995 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS       = gpfshdfs.o
 
 PG_CPPFLAGS = -I$(libpq_srcdir)
 PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_system -lboost_date_time -lstdc++ -lhdfs3
 
 ifdef USE_PGXS
 PGXS := $(shell pg_config --pgxs)


[23/50] incubator-hawq git commit: Revert "HAWQ-529. Allocate resource for udf in resource negotiator."

Posted by sh...@apache.org.
Revert "HAWQ-529. Allocate resource for udf in resource negotiator."

This reverts commit 8e38e844f0d2a1c45f7d5cd25a8f5b639c8b6fb7.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fd202277
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fd202277
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fd202277

Branch: refs/heads/HAWQ-459
Commit: fd2022772d1f39e5f205347a4c0cfd0a0f04021f
Parents: bc065a5
Author: hzhang2 <zh...@163.com>
Authored: Thu Mar 17 09:19:04 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Thu Mar 17 09:19:04 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c    | 194 ++++++++++--------------------
 src/backend/executor/spi.c           |  72 ++---------
 src/backend/optimizer/plan/planner.c |  57 +--------
 src/backend/optimizer/util/clauses.c |  11 +-
 src/backend/optimizer/util/walkers.c |   1 -
 src/include/cdb/cdbdatalocality.h    |  19 +--
 src/include/executor/spi.h           |   7 --
 src/include/optimizer/planner.h      |   4 -
 8 files changed, 76 insertions(+), 289 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index d15d6b0..c1f829b 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,9 +33,6 @@
 #include "access/filesplit.h"
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
-#include "catalog/catquery.h"
-#include "catalog/pg_inherits.h"
-#include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
 #include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
@@ -44,7 +41,6 @@
 #include "utils/tqual.h"
 #include "utils/memutils.h"
 #include "executor/execdesc.h"
-#include "executor/spi.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "optimizer/walkers.h"
@@ -58,7 +54,11 @@
 #include "catalog/pg_proc.h"
 #include "postgres.h"
 #include "resourcemanager/utils/hashtable.h"
+#include "catalog/pg_inherits.h"
+//#include "utils/misc/guc.c"
 
+#define PRONAME 1
+#define PROISAGG 5
 /* We need to build a mapping from host name to host index */
 
 extern bool		optimizer; /* Enable the optimizer */
@@ -372,7 +372,7 @@ static Block_Host_Index * update_data_dist_stat(
 static HostDataVolumeInfo *search_host_in_stat_context(
 		split_to_segment_mapping_context *context, char *hostname);
 
-static bool IsBuildInFunction(Oid funcOid);
+static bool IsAggFunction(char* funcName);
 
 static bool allocate_hash_relation(Relation_Data* rel_data,
 		Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -630,36 +630,39 @@ static void collect_range_tables(Query *query, List* full_range_table,
 /*
  *
  */
-static bool IsBuildInFunction(Oid foid) {
+static bool IsAggFunction(char* funcName) {
+	if (funcName == NULL) {
+		return false;
+	}
+	Relation pg_proc_rel;
+	TupleDesc pg_proc_dsc;
+	HeapTuple tuple;
+	SysScanDesc pg_proc_scan;
 
-	cqContext  *pcqCtx;
-	HeapTuple readtup = NULL;
-	HeapTuple	procedureTuple;
-	Form_pg_proc procedureStruct;
+	pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
+	pg_proc_dsc = RelationGetDescr(pg_proc_rel);
+	ScanKeyData skey;
 
-	/*
-	 * get the procedure tuple corresponding to the given function Oid
-	 */
-	pcqCtx = caql_beginscan(
-			NULL,
-			cql("SELECT * FROM pg_proc "
-				" WHERE oid = :1 ",
-				ObjectIdGetDatum(foid)));
-
-	procedureTuple = caql_getnext(pcqCtx);
-
-	if (!HeapTupleIsValid(procedureTuple))
-		elog(ERROR, "cache lookup failed for function %u", foid);
-	procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
-	caql_endscan(pcqCtx);
-	/* we treat proc namespace = 11 to build in function.*/
-	if (procedureStruct->pronamespace == 11) {
-		return true;
-	} else {
-		return false;
+	ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
+	F_NAMEEQ, CStringGetDatum(funcName));
+
+	pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
+			ActiveSnapshot, 1, &skey);
+	while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+
+		bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
+		systable_endscan(pg_proc_scan);
+		heap_close(pg_proc_rel, AccessShareLock);
+		if (isAgg) {
+			return true;
+		} else {
+			return false;
+		}
 	}
+	systable_endscan(pg_proc_scan);
+	heap_close(pg_proc_rel, AccessShareLock);
+	return true;
 }
-
 /*
  *
  */
@@ -673,6 +676,25 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
 	foreach(old_lc, *range_tables)
 	{
 		RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
+		if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
+			*isTableFunctionExists = true;
+		}
+		if (entry->rtekind == RTE_SUBQUERY) {
+			Query* subQuery = entry->subquery;
+			ListCell *lc;
+			foreach(lc, subQuery->targetList)
+			{
+				TargetEntry *te = (TargetEntry *) lfirst(lc);
+				bool isAggFunc = IsAggFunction(te->resname);
+				// if target list of subquery contains non aggregate function,
+				// then we consider the query contains and use default_segment_num guc
+				// as the number of virtual segment
+				if (!isAggFunc) {
+					*isTableFunctionExists = true;
+				}
+			}
+
+		}
 		if (entry->rtekind != RTE_RELATION) {
 			continue;
 		}
@@ -3936,54 +3958,11 @@ static void cleanup_allocation_algorithm(
 }
 
 /*
- * udf_collector_walker: the routine to file udfs.
- */
-bool udf_collector_walker(Node *node,
-		udf_collector_context *context) {
-	if (node == NULL) {
-		return false;
-	}
-
-	if (IsA(node, Query)) {
-		return query_tree_walker((Query *) node, udf_collector_walker,
-				(void *) context,
-				QTW_EXAMINE_RTES);
-	}
-
-	/*For Aggref, we don't consider it as udf.*/
-
-	if(IsA(node,FuncExpr)){
-		if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
-			context->udf_exist = true;
-		}
-		return false;
-	}
-
-	return expression_tree_walker(node, udf_collector_walker,
-			(void *) context);
-
-	return false;
-}
-
-/*
- * find_udf: collect all udf, and store them into the udf_collector_context.
- */
-void find_udf(Query *query, udf_collector_context *context) {
-
-	query_tree_walker(query, udf_collector_walker, (void *) context,
-	QTW_EXAMINE_RTES);
-
-	return;
-}
-
-
-/*
  * calculate_planner_segment_num
- * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
  */
 SplitAllocResult *
 calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
+		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
 	SplitAllocResult *result = NULL;
 	QueryResource *resource = NULL;
 	QueryResourceParameters *resource_parameters = NULL;
@@ -3992,14 +3971,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	List *alloc_result = NIL;
 	split_to_segment_mapping_context context;
 
-	int planner_segments = 0; /*virtual segments number for explain statement */
+	int planner_segments = -1; /*virtual segments number for explain statement */
 
 	result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
 	result->resource = NULL;
 	result->resource_parameters = NULL;
 	result->alloc_results = NIL;
 	result->relsType = NIL;
-	result->planner_segments = 0;
+	result->planner_segments = -1;
 	result->datalocalityInfo = makeStringInfo();
 
 	/* fake data locality */
@@ -4016,7 +3995,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		result->resource_parameters = NULL;
 		result->alloc_results = NIL;
 		result->relsType = NIL;
-		result->planner_segments = 0;
+		result->planner_segments = -1;
 		return result;
 	}
 
@@ -4041,11 +4020,6 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		 * 5 data size of random "from" relation
 		 */
 
-		udf_collector_context udf_context;
-		udf_context.udf_exist = false;
-
-		find_udf(query, &udf_context);
-		isTableFunctionExists = udf_context.udf_exist;
 		/*convert range table list to oid list and check whether table function exists
 		 *we keep a full range table list and a range table list without result relation separately
 		 */
@@ -4072,15 +4046,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 		/*use inherit resource*/
 		if (resourceLife == QRL_INHERIT) {
-
-			if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
-			{
-				resource = NULL;
-			}
-			else
-			{
-				resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
-			}
+			resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
 
 			saveQueryResourceParameters(
 							resource_parameters,  /* resource_parameters */
@@ -4231,11 +4197,6 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				maxTargetSegmentNumber = enforce_virtual_segment_number;
 				minTargetSegmentNumber = enforce_virtual_segment_number;
 			}
-			/* in PBE mode, the execute should use the same vseg number. */
-			if(fixedVsegNum > 0 ){
-				maxTargetSegmentNumber = fixedVsegNum;
-				minTargetSegmentNumber = fixedVsegNum;
-			}
 			uint64_t before_rm_allocate_resource = gettime_microsec();
 
 			/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4243,40 +4204,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			mincost <<= 20;
 			int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
 			if (QRL_NONE != resourceLife) {
-
-				if (SPI_IsInPrepare())
-				{
-					resource = NULL;
-					/*
-					 * prepare need to get resource quota from RM
-					 * and pass quota(planner_segments) to Orca or Planner to generate plan
-					 * the following executes(in PBE) should reallocate the same number
-					 * of resources.
-					 */
-					uint32 seg_num;
-					uint32 seg_num_min;
-					uint32 seg_memory_mb;
-					double seg_core;
-
-					GetResourceQuota(maxTargetSegmentNumber,
-					                 minTargetSegmentNumber,
-					                 &seg_num,
-					                 &seg_num_min,
-					                 &seg_memory_mb,
-					                 &seg_core);
-
-					planner_segments = seg_num;
-					minTargetSegmentNumber = planner_segments;
-					maxTargetSegmentNumber = planner_segments;
-				}
-				else
-				{
-					resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
-					                            maxTargetSegmentNumber,
-					                            minTargetSegmentNumber,
-					                            context.host_context.hostnameVolInfos,
-					                            context.host_context.size);
-				}
+				resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+						maxTargetSegmentNumber, minTargetSegmentNumber,
+						context.host_context.hostnameVolInfos, context.host_context.size);
 
 				saveQueryResourceParameters(
 								resource_parameters,                   /* resource_parameters */
@@ -4306,7 +4236,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 			if (resource == NULL) {
 				result->resource = NULL;
-				result->resource_parameters = resource_parameters;
+				result->resource_parameters = NULL;
 				result->alloc_results = NIL;
 				result->relsType = NIL;
 				result->planner_segments = planner_segments;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 8e7645c..07812aa 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,7 +46,6 @@
 #include "cdb/memquota.h"
 #include "executor/nodeFunctionscan.h"
 #include "nodes/stack.h"
-#include "cdb/cdbdatalocality.h"
 
 extern char *savedSeqServerHost;
 extern int savedSeqServerPort;
@@ -75,7 +74,6 @@ static int	_SPI_curid = -1;
 
 static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
 static char *_QD_currently_prepared_stmt = NULL;
-static int SPI_prepare_depth = 0;
 
 static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
 
@@ -108,32 +106,6 @@ static bool _SPI_checktuples(void);
 
 /* =================== interface functions =================== */
 
-bool SPI_IsInPrepare(void)
-{
-	if (SPI_prepare_depth > 0)
-	{
-		return true;
-	}
-	else if (SPI_prepare_depth < 0)
-	{
-		elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
-		            SPI_prepare_depth);
-	}
-
-	return false;
-}
-
-void SPI_IncreasePrepareDepth(void)
-{
-	SPI_prepare_depth++;
-}
-
-void SPI_DecreasePrepareDepth(void)
-{
-	SPI_prepare_depth--;
-}
-
-
 int
 SPI_connect(void)
 {
@@ -594,8 +566,6 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 	_SPI_plan	plan;
 	_SPI_plan  *result;
 
-	SPI_IncreasePrepareDepth();
-
 	if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
 	{
 		SPI_result = SPI_ERROR_ARGUMENT;
@@ -621,13 +591,9 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 
 		/* copy plan to procedure context */
 		result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
-
-		SPI_DecreasePrepareDepth();
 	}
 	PG_CATCH();
 	{
-		SPI_DecreasePrepareDepth();
-
 		_SPI_end_call(true);
 		PG_RE_THROW();
 	}
@@ -1853,37 +1819,13 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 					     (stmt->resource == NULL) &&
 					     (stmt->resource_parameters != NULL) )
 					{
-						SplitAllocResult *allocResult = NULL;
-
-						/* If this is a parallel plan. */
-						if (stmt->planTree->dispatch == DISPATCH_PARALLEL)
-						{
-							/*
-							 * Now, we want to allocate resource.
-							 */
-							allocResult = calculate_planner_segment_num(queryTree,
-							                                            stmt->resource_parameters->life,
-							                                            stmt->rtable,
-							                                            stmt->intoPolicy,
-							                                            stmt->nMotionNodes + stmt->nInitPlans + 1,
-							                                            stmt->resource_parameters->min_target_segment_num);
-
-							Assert(allocResult);
-
-							if(stmt->resource !=NULL)
-							{
-								pfree(stmt->resource);
-							}
-							stmt->resource = allocResult->resource;
-							if(stmt->scantable_splits !=NULL)
-							{
-								list_free_deep(stmt->scantable_splits);
-							}
-							stmt->scantable_splits = allocResult->alloc_results;
-							stmt->planner_segments = allocResult->planner_segments;
-							stmt->datalocalityInfo = allocResult->datalocalityInfo;
-							pfree(allocResult);
-						}
+						stmt->resource = AllocateResource(stmt->resource_parameters->life,
+						                        stmt->resource_parameters->slice_size,
+						                        stmt->resource_parameters->iobytes,
+						                        stmt->resource_parameters->max_target_segment_num,
+						                        stmt->resource_parameters->min_target_segment_num,
+						                        stmt->resource_parameters->vol_info,
+						                        stmt->resource_parameters->vol_info_size);
 					}
 
 					originalStmt->resource = NULL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index c6be59a..be057e0 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,8 +91,6 @@ planner_hook_type planner_hook = NULL;
 
 ParamListInfo PlannerBoundParamList = NULL;		/* current boundParams */
 
-static int PlanningDepth = 0;		/* Planning depth */
-
 /* Expression kind codes for preprocess_expression */
 #define EXPRKIND_QUAL			0
 #define EXPRKIND_TARGET			1
@@ -167,32 +165,6 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
 static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
 bool is_dummy_plan(Plan *plan);
 
-
-bool is_in_planning_phase(void)
-{
-	if (PlanningDepth > 0)
-	{
-		return true;
-	}
-	else if (PlanningDepth < 0)
-	{
-		elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
-	}
-
-	return false;
-}
-
-void increase_planning_depth(void)
-{
-	PlanningDepth++;
-}
-
-void decrease_planning_depth(void)
-{
-	PlanningDepth--;
-}
-
-
 #ifdef USE_ORCA
 /**
  * Logging of optimization outcome
@@ -313,7 +285,7 @@ planner(Query *parse, int cursorOptions,
 	PlannedStmt *result = NULL;
 	instr_time	starttime, endtime;
 	ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
-	SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
+	SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
 	ppResult->saResult = initResult;
 	ppResult->stmt = NULL;
 	static int plannerLevel = 0;
@@ -328,8 +300,6 @@ planner(Query *parse, int cursorOptions,
 	 * resource to run this query. After gaining the resource, we can perform the
 	 * actual optimization.
 	 */
-	increase_planning_depth();
-
 	plannerLevel++;
 	if (!resourceNegotiateDone)
 	{
@@ -339,8 +309,6 @@ planner(Query *parse, int cursorOptions,
       {
         resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
 
-		decrease_planning_depth();
-
 		if(ppResult->stmt && ppResult->stmt->planTree)
 		{
 			isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -350,8 +318,6 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  PG_CATCH();
 	  {
-		decrease_planning_depth();
-
 		if ((ppResult != NULL))
 		{
 		  pfree(ppResult);
@@ -464,7 +430,7 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  else
 	  {
-	    gp_segments_for_planner = 0;
+	    gp_segments_for_planner = -1;
 	  }
 	  SetActiveQueryResource(savedQueryResource);
 	  if ((ppResult != NULL))
@@ -484,7 +450,7 @@ planner(Query *parse, int cursorOptions,
 		}
 		else
 		{
-			gp_segments_for_planner = 0;
+			gp_segments_for_planner = -1;
 		}
 		SetActiveQueryResource(savedQueryResource);
 
@@ -504,7 +470,6 @@ planner(Query *parse, int cursorOptions,
 	return result;
 }
 
-
 /*
  * The new framework for HAWQ 2.0 query optimizer
  */
@@ -515,8 +480,6 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
   PlannedStmt *plannedstmt = NULL;
   do
   {
-  		udf_collector_context udf_context;
-  		udf_context.udf_exist = false;
     SplitAllocResult *allocResult = NULL;
     Query *my_parse = copyObject(parse);
     ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -532,24 +495,12 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
        */
       allocResult = calculate_planner_segment_num(my_parse, resourceLife,
                                           plannedstmt->rtable, plannedstmt->intoPolicy,
-                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
-                                          -1);
+                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
 
       Assert(allocResult);
 
       (*result)->saResult = *allocResult;
       pfree(allocResult);
-    }else{
-    		find_udf(my_parse, &udf_context);
-    		if(udf_context.udf_exist){
-    			if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
-    				int64 mincost = min_cost_for_each_query;
-    				mincost <<= 20;
-    				int avgSliceNum = 3;
-    				(*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
-    						GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
-    			}
-    		}
     }
   } while (0);
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a3d67f2..a80737f 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,15 +2831,8 @@ simplify_function(Oid funcid, Oid result_type, List *args,
 	if (!HeapTupleIsValid(func_tuple))
 		elog(ERROR, "cache lookup failed for function %u", funcid);
 
-	if (is_in_planning_phase())
-	{
-		newexpr = NULL;
-	}
-	else
-	{
-		newexpr = evaluate_function(funcid, result_type, args,
-		                            func_tuple, context);
-	}
+	newexpr = evaluate_function(funcid, result_type, args,
+								func_tuple, context);
 
 	if (large_const(newexpr, context->max_size))
 	{

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index a8222fb..6f1ee51 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,7 +162,6 @@ expression_tree_walker(Node *node,
 		case T_PartBoundExpr:
 		case T_PartBoundInclusionExpr:
 		case T_PartBoundOpenExpr:
-		case T_RangeTblEntry	:
 			/* primitive node types with no expression subnodes */
 			break;
 		case T_Aggref:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 67375d7..95f2bd5 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -49,13 +49,6 @@ typedef struct SplitAllocResult
 } SplitAllocResult;
 
 /*
- * structure containing all relation range table entries.
- */
-typedef struct udf_collector_context {
-	bool udf_exist;
-} udf_collector_context;
-
-/*
  * structure containing rel and type when execution
  */
 typedef struct CurrentRelType {
@@ -93,17 +86,7 @@ void saveQueryResourceParameters(
  * we calculate the appropriate planner segment_num.
  */
 SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-                                                List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
-
-/*
- * udf_collector_walker: the routine to file udfs.
- */
-bool udf_collector_walker(Node *node,	udf_collector_context *context);
-
-/*
- * find_udf: collect all udf, and store them into the udf_collector_context.
- */
-void find_udf(Query *query, udf_collector_context *context);
+                                                List *rtable, GpPolicy *intoPolicy, int sliceNum);
 
 FILE *fp;
 FILE *fpaoseg;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 7b4ba1b..867daf5 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,11 +155,4 @@ extern uint64 SPI_GetMemoryReservation(void);
 extern void SPI_ReserveMemory(uint64 mem_reserved);
 extern bool SPI_IsMemoryReserved(void);
 
-/**
- * Query resource related routines.
- */
-extern bool SPI_IsInPrepare(void);
-extern void SPI_IncreasePrepareCounter(void);
-extern void SPI_DecreasePrepareCounter(void);
-
 #endif   /* SPI_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd202277/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index cc2eb7d..48de348 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,8 +50,4 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
 								   double dNumGroups, 
 								   AggClauseCounts *agg_counts);
 
-extern bool is_in_planning_phase(void);
-extern void increase_planning_depth(void);
-extern void decrease_planning_depth(void);
-
 #endif   /* PLANNER_H */


[05/50] incubator-hawq git commit: HAWQ-524. Modify processing for cdbDispatchResult->resultBuf when creation and free

Posted by sh...@apache.org.
HAWQ-524. Modify processing for cdbDispatchResult->resultBuf when creation and free


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ac26f48f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ac26f48f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ac26f48f

Branch: refs/heads/HAWQ-459
Commit: ac26f48fd44af17c26cee0ce48a44bf50ddeb9ca
Parents: 4b53168
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 17:47:42 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Tue Mar 15 09:38:40 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdispatchresult.c | 19 +++++++++++--------
 src/backend/cdb/executormgr.c       |  4 +++-
 2 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/cdbdispatchresult.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdispatchresult.c b/src/backend/cdb/cdbdispatchresult.c
index b036b26..9ba90c0 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -183,15 +183,18 @@ void
 cdbdisp_resetResult(CdbDispatchResult  *dispatchResult)
 {
     PQExpBuffer buf = dispatchResult->resultbuf;
-    PGresult  **begp = (PGresult **)buf->data;
-    PGresult  **endp = (PGresult **)(buf->data + buf->len);
-    PGresult  **p;
-
-    /* Free the PGresult objects. */
-    for (p = begp; p < endp; ++p)
+    if (buf != NULL)
     {
-        Assert(*p != NULL);
-        PQclear(*p);
+        PGresult  **begp = (PGresult **)buf->data;
+        PGresult  **endp = (PGresult **)(buf->data + buf->len);
+        PGresult  **p;
+
+        /* Free the PGresult objects. */
+        for (p = begp; p < endp; ++p)
+        {
+            Assert(*p != NULL);
+            PQclear(*p);
+        }
     }
 
     /* Reset summary indicators. */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ac26f48f/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 88dd32f..47f5bbe 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -220,7 +220,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
 	executor->refResult = cdbdisp_makeResult(dispatch_get_results(data),
 											executor->desc,
 											dispatch_get_task_identity(task)->slice_id);
-	Assert(executor->refResult != NULL);
+
+	if (executor->refResult == NULL)
+	  return false;
 
 	/* Transfer any connection errors from segdbDesc. */
 	executormgr_merge_error(executor);


[24/50] incubator-hawq git commit: HAWQ-549. Ignore empty 'pg_log' folder while doing standby data directory check

Posted by sh...@apache.org.
HAWQ-549. Ignore empty 'pg_log' folder while doing standby data directory check


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aef7bd2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aef7bd2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aef7bd2f

Branch: refs/heads/HAWQ-459
Commit: aef7bd2f3465684d84b7b9ee515aee39f4346780
Parents: fd20227
Author: rlei <rl...@pivotal.io>
Authored: Wed Mar 16 16:53:12 2016 +0800
Committer: rlei <rl...@pivotal.io>
Committed: Thu Mar 17 10:36:09 2016 +0800

----------------------------------------------------------------------
 tools/bin/lib/hawqinit.sh | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aef7bd2f/tools/bin/lib/hawqinit.sh
----------------------------------------------------------------------
diff --git a/tools/bin/lib/hawqinit.sh b/tools/bin/lib/hawqinit.sh
index dab0b34..43ffdef 100755
--- a/tools/bin/lib/hawqinit.sh
+++ b/tools/bin/lib/hawqinit.sh
@@ -473,8 +473,13 @@ check_data_directorytory() {
         ${MKDIR} -p ${default_sdd}
     fi
     # Check if data directory already exist and clean.
+    if [ "${hawq_data_directory}" = "" ]; then
+        LOG_MSG "[ERROR]:-Data directory path is not valid value on ${host_name}" verbose
+        exit 1
+    fi
+
     if [ -d ${hawq_data_directory} ]; then
-        if [ "$(ls -A ${hawq_data_directory})" ] && [ "${hawq_data_directory}" != "" ]; then
+        if [ "$(ls -A ${hawq_data_directory})" ]; then
              LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
              exit 1
         fi
@@ -484,6 +489,37 @@ check_data_directorytory() {
     fi
 }
 
+check_standby_data_directorytory() {
+    # If it's default directory, create it if not exist.
+    default_mdd=~/hawq-data-directory/masterdd
+    default_sdd=~/hawq-data-directory/segmentdd
+    if [ "${hawq_data_directory}" = "${default_mdd}" ]; then
+        ${MKDIR} -p ${default_mdd}
+    elif [ "${hawq_data_directory}" = "${default_sdd}" ]; then
+        ${MKDIR} -p ${default_sdd}
+    fi
+    # Check if data directory already exist and clean.
+    if [ "${hawq_data_directory}" = "" ]; then
+        LOG_MSG "[ERROR]:-Data directory path is not valid value on ${host_name}" verbose
+        exit 1
+    fi
+
+    if [ -d "${hawq_data_directory}" ]; then
+        if [ "$(ls -A ${hawq_data_directory}| ${GREP} -v pg_log)" ]; then
+            LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
+            exit 1
+        else
+            if [ -d ${hawq_data_directory}/pg_log ] && [ "$(ls -A ${hawq_data_directory}/pg_log)" ]; then
+                 LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} is not empty on ${host_name}" verbose
+                 exit 1
+            fi
+        fi
+    else
+        LOG_MSG "[ERROR]:-Data directory ${hawq_data_directory} does not exist on ${host_name}, please create it" verbose
+        exit 1
+    fi
+}
+
 check_temp_directory() {
     # Check if temp directory exist.
     for tmp_dir in ${tmp_dir_list}; do
@@ -504,7 +540,7 @@ if [ ${object_type} == "master" ]; then
     check_temp_directory
     master_init
 elif [ ${object_type} == "standby" ]; then
-    check_data_directorytory
+    check_standby_data_directorytory
     standby_init
 elif [ ${object_type} == "segment" ]; then
     check_data_directorytory


[07/50] incubator-hawq git commit: HAWQ-404. Add sort during INSERT of append only AO partition tables

Posted by sh...@apache.org.
HAWQ-404. Add sort during INSERT of append only AO partition tables


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2a150660
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2a150660
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2a150660

Branch: refs/heads/HAWQ-459
Commit: 2a150660417bd61db1d0b30c7e7bce9222e2fda1
Parents: ac26f48 6e6890a
Author: ivan <iw...@pivotal.io>
Authored: Tue Mar 15 09:53:24 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Tue Mar 15 09:53:24 2016 +0800

----------------------------------------------------------------------
 src/backend/executor/execDML.c                  |  99 +++++++++------
 src/backend/executor/execMain.c                 |  13 +-
 src/backend/executor/execUtils.c                |   2 +-
 src/backend/executor/nodeDML.c                  |   2 +-
 src/backend/gpopt/ivy.xml                       |   2 +-
 .../gpopt/translate/CTranslatorDXLToPlStmt.cpp  |   2 +
 .../translate/CTranslatorRelcacheToDXL.cpp      |   9 ++
 src/backend/gpopt/utils/COptTasks.cpp           |   4 +-
 src/backend/nodes/copyfuncs.c                   |   1 +
 src/backend/nodes/outfast.c                     |   1 +
 src/backend/nodes/outfuncs.c                    |   1 +
 src/backend/nodes/readfast.c                    |   1 +
 src/backend/utils/misc/guc.c                    |  11 ++
 src/include/executor/execDML.h                  |   3 +-
 src/include/gpopt/translate/CTranslatorUtils.h  |   4 -
 src/include/nodes/execnodes.h                   |   7 +-
 src/include/nodes/plannodes.h                   |   1 +
 src/include/utils/guc.h                         |   1 +
 src/test/regress/expected/goh_partition.out     | 126 +++++++++++++++++++
 .../expected/gpsql_alter_table_optimizer.out    |   2 +-
 src/test/regress/expected/insert_optimizer.out  |   2 +-
 src/test/regress/sql/goh_partition.sql          |  68 ++++++++++
 22 files changed, 304 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a150660/src/backend/utils/misc/guc.c
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2a150660/src/include/utils/guc.h
----------------------------------------------------------------------


[20/50] incubator-hawq git commit: HAWQ-539. Add fault injection for dispatcher QD site

Posted by sh...@apache.org.
HAWQ-539. Add fault injection for dispatcher QD site


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1a75a2a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1a75a2a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1a75a2a7

Branch: refs/heads/HAWQ-459
Commit: 1a75a2a7c191ce16371d9e41d62786228dfa5774
Parents: 4d1a854
Author: Chunling Wang <wa...@ChunlingdeMacBook-Pro.local>
Authored: Tue Mar 15 10:15:03 2016 +0800
Committer: ivan <iw...@pivotal.io>
Committed: Wed Mar 16 14:09:11 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdispatchresult.c            |  9 +++++++++
 src/backend/cdb/dispatcher.c                   |  9 +++++++++
 src/backend/cdb/workermgr.c                    | 10 ++++++++++
 src/backend/utils/misc/faultinjector.c         |  9 ++++++++-
 src/include/utils/faultinjector.h              |  2 ++
 tools/bin/gppylib/programs/clsInjectFault.py   |  5 ++++-
 tools/bin/hawqpylib/programs/clsInjectFault.py |  5 ++++-
 7 files changed, 46 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/cdbdispatchresult.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdispatchresult.c b/src/backend/cdb/cdbdispatchresult.c
index 9ba90c0..d97dda5 100644
--- a/src/backend/cdb/cdbdispatchresult.c
+++ b/src/backend/cdb/cdbdispatchresult.c
@@ -33,6 +33,7 @@
 
 #include "lib/stringinfo.h"         /* StringInfoData */
 #include "utils/guc.h"              /* log_min_messages */
+#include "utils/faultinjector.h"
 
 #include "cdb/cdbconn.h"            /* SegmentDatabaseDescriptor */
 #include "cdb/cdbpartition.h"
@@ -110,6 +111,14 @@ cdbdisp_makeResult(struct CdbDispatchResults           *meleeResults,
     meleeIndex = meleeResults->resultCount++;
     dispatchResult = &meleeResults->resultArray[meleeIndex];
 
+#ifdef FAULT_INJECTOR
+				FaultInjector_InjectFaultIfSet(
+											   CreateCdbDispathResultObject,
+											   DDLNotSpecified,
+											   "",	// databaseName
+											   ""); // tableName
+#endif
+
     /* Initialize CdbDispatchResult. */
     dispatchResult->meleeResults = meleeResults;
     dispatchResult->meleeIndex = meleeIndex;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 20e97ab..5d7105c 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -49,6 +49,7 @@
 #include "utils/memutils.h"	/* GetMemoryChunkContext */
 #include "cdb/cdbsrlz.h"	/* serializeNode */
 #include "utils/datum.h"	/* datumGetSize */
+#include "utils/faultinjector.h"
 #include "utils/lsyscache.h"	/* get_typlenbyval */
 #include "miscadmin.h"		/* CHECK_FOR_INTERRUPTS */
 #include "tcop/tcopprot.h"	/* ResetUsage */
@@ -963,6 +964,14 @@ dispatcher_bind_executor(DispatchData *data)
 				continue;
 			}
 
+#ifdef FAULT_INJECTOR
+				FaultInjector_InjectFaultIfSet(
+											   ConnectionFailAfterGangCreation,
+											   DDLNotSpecified,
+											   "",	// databaseName
+											   ""); // tableName
+#endif
+
 			if (!executormgr_bind_executor_task(data, executor, desc, task, slice))
 				return false;
 			data->num_of_cached_executors++;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/cdb/workermgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/workermgr.c b/src/backend/cdb/workermgr.c
index eaf3060..2a31d29 100644
--- a/src/backend/cdb/workermgr.c
+++ b/src/backend/cdb/workermgr.c
@@ -35,6 +35,8 @@
 #include "cdb/cdbgang.h"		/* gp_pthread_create */
 #include "miscadmin.h"			/* TODO: InterruptPending */
 
+#include "utils/faultinjector.h"
+
 
 /*
  * This structure abstract the general job.
@@ -114,6 +116,14 @@ workermgr_submit_job(WorkerMgrState *state,
 		i++;
 		worker_mgr_thread->func = func;
 
+#ifdef FAULT_INJECTOR
+				FaultInjector_InjectFaultIfSet(
+											   WorkerManagerSubmitJob,
+											   DDLNotSpecified,
+											   "",	// databaseName
+											   ""); // tableName
+#endif
+
 		worker_mgr_thread->thread_ret = gp_pthread_create(&worker_mgr_thread->thread, workermgr_thread_func, worker_mgr_thread, "submit_plan_to_qe");
 		if (worker_mgr_thread->thread_ret)
 			goto error_cleanup;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/backend/utils/misc/faultinjector.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/faultinjector.c b/src/backend/utils/misc/faultinjector.c
index 44b7462..f5ab5bc 100644
--- a/src/backend/utils/misc/faultinjector.c
+++ b/src/backend/utils/misc/faultinjector.c
@@ -281,8 +281,12 @@ FaultInjectorIdentifierEnumToString[] = {
 		/* inject fault after dispatcher thread creation*/
 /*	_("dispatch_wait"),
 		 inject fault after dispatcher wait for results from segments*/
-	_("connection_fail_after_gang"),
+	_("connection_fail_after_gang_creation"),
 		/* inject fault after gang thread creation, set connection null*/
+	_("create_cdb_dispath_result_object"),
+		/* inject fault when create cdb dispatch result object, set out of memory */
+	_("worker_manager_submit_job"),
+		/* inject fault when worker manager submit job , set error*/
 /*	_("make_dispatch_thread"),
 		 inject fault when initialing memory structure for dispatcher thread*/
 	_("before_dispatch"),
@@ -1070,6 +1074,9 @@ FaultInjector_NewHashEntry(
 		case DtmXLogDistributedCommit:
 		case AnalyzeSubxactError:
 		case OptTaskAllocateStringBuffer:
+		case ConnectionFailAfterGangCreation:
+		case CreateCdbDispathResultObject:
+		case WorkerManagerSubmitJob:
 
 			/* These faults are designed for master. */
 			if(!AmIMaster())

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/src/include/utils/faultinjector.h
----------------------------------------------------------------------
diff --git a/src/include/utils/faultinjector.h b/src/include/utils/faultinjector.h
index f197d2e..ca9c278 100644
--- a/src/include/utils/faultinjector.h
+++ b/src/include/utils/faultinjector.h
@@ -188,6 +188,8 @@ typedef enum FaultInjectorIdentifier_e {
 	ConnectionFailAfterGangCreation,
 
 /*	MakeDispatchThread,*/
+	CreateCdbDispathResultObject,
+	WorkerManagerSubmitJob,
 
 	BeforeDispatch,
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/gppylib/programs/clsInjectFault.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/programs/clsInjectFault.py b/tools/bin/gppylib/programs/clsInjectFault.py
index be6d5ce..6f097d8 100644
--- a/tools/bin/gppylib/programs/clsInjectFault.py
+++ b/tools/bin/gppylib/programs/clsInjectFault.py
@@ -405,7 +405,10 @@ class GpInjectFaultProgram:
                   "analyze_subxact_error (inject an error during analyze)," \
                   "opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \
                   "runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \
-                  "all (affects all faults injected, used for 'status' and 'reset'), ") 
+                  "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \
+				  "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memeory)" \
+				  "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \
+				  "all (affects all faults injected, used for 'status' and 'reset'), ")
         addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string",
                          metavar="ddlStatement",
                          help="The DDL statement on which fault should be set and triggered " \

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1a75a2a7/tools/bin/hawqpylib/programs/clsInjectFault.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/programs/clsInjectFault.py b/tools/bin/hawqpylib/programs/clsInjectFault.py
index af4d9df..8992f9d 100644
--- a/tools/bin/hawqpylib/programs/clsInjectFault.py
+++ b/tools/bin/hawqpylib/programs/clsInjectFault.py
@@ -413,7 +413,10 @@ class HAWQInjectFaultProgram:
                   "analyze_subxact_error (inject an error during analyze)," \
                   "opt_task_allocate_string_buffer (inject fault while allocating string buffer), " \
                   "runaway_cleanup (inject fault before starting the cleanup for a runaway query)" \
-                  "all (affects all faults injected, used for 'status' and 'reset'), ") 
+                  "connection_fail_after_gang_creation (inject fault after gang thread creation, set connection null)" \
+				  "create_cdb_dispath_result_object (inject fault when create cdb dispatch result object, set out of memory)" \
+                  "worker_manager_submit_job (inject fault when worker manager submit job , set error)" \
+				  "all (affects all faults injected, used for 'status' and 'reset'), ")
         addTo.add_option("-c", "--ddl_statement", dest="ddlStatement", type="string",
                          metavar="ddlStatement",
                          help="The DDL statement on which fault should be set and triggered " \


[36/50] incubator-hawq git commit: HAWQ-532. Optimise vseg number for copy to statement.

Posted by sh...@apache.org.
HAWQ-532. Optimise vseg number for copy to statement.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c29d6476
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c29d6476
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c29d6476

Branch: refs/heads/HAWQ-459
Commit: c29d647658e5e4219a370b4869f68cdcd577ab65
Parents: eaaf945
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:30:20 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbmutate.c          |  2 +-
 src/backend/commands/copy.c          | 96 ++++++++++++++++++++++++++++---
 src/backend/commands/prepare.c       |  2 +-
 src/backend/optimizer/util/relnode.c |  2 +-
 src/backend/parser/analyze.c         |  8 +--
 src/backend/postmaster/identity.c    |  2 +-
 src/backend/utils/misc/guc.c         | 12 ++--
 src/include/postmaster/identity.h    |  2 +-
 8 files changed, 103 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/cdb/cdbmutate.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmutate.c b/src/backend/cdb/cdbmutate.c
index 6959c64..9a998da 100644
--- a/src/backend/cdb/cdbmutate.c
+++ b/src/backend/cdb/cdbmutate.c
@@ -312,7 +312,7 @@ apply_motion(PlannerInfo *root, Plan *plan, Query *query)
 				
 					targetPolicy = palloc0(sizeof(GpPolicy)- sizeof(targetPolicy->attrs) + maxattrs * sizeof(targetPolicy->attrs[0]));
 					targetPolicy->nattrs = 0;
-					targetPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, GetRandomDistPartitionNum());
+					targetPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(query->intoClause->rel, GetDefaultPartitionNum());
 					targetPolicy->ptype = POLICYTYPE_PARTITIONED;
 					
 					/* Find out what the flow is partitioned on */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index bcd5384..2f4cf39 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,6 +91,11 @@
 #include "postmaster/autovacuum.h"
 #include "cdb/dispatcher.h"
 
+/*
+ * in dbsize.c
+ */
+extern int64 calculate_relation_size(Relation rel);
+
 /* DestReceiver for COPY (SELECT) TO */
 typedef struct
 {
@@ -137,6 +142,7 @@ static void copy_in_error_callback(void *arg);
 static void CopyInitPartitioningState(EState *estate);
 static void CopyInitDataParser(CopyState cstate);
 static bool CopyCheckIsLastLine(CopyState cstate);
+static int calculate_virtual_segment_number(List* candidateRelations);
 
 /* ==========================================================================
  * The follwing macros aid in major refactoring of data processing code (in
@@ -1556,7 +1562,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 
 			target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
 			Assert(target_policy);
-			target_segment_num = target_policy->bucketnum;
+			/*
+			 * For hash table we use table bucket number to request vsegs
+			 * For random table, we use a fixed GUC value to request vsegs.
+			 */
+			if(target_policy->nattrs > 0){
+				target_segment_num = target_policy->bucketnum;
+			}
+			else{
+				target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
+			}
 			pfree(target_policy);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1749,6 +1764,63 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 }
 
 /*
+ * calculate virtual segment number for copy statement.
+ * if there is hash distributed relations exist, use the max bucket number.
+ * if all relation are random, use the data size to determine vseg number.
+ */
+static int calculate_virtual_segment_number(List* candidateOids) {
+	ListCell* le1;
+	int vsegNumber = 1;
+	int64 totalDataSize = 0;
+	bool isHashRelationExist = false;
+	int maxHashBucketNumber = 0;
+
+	foreach (le1, candidateOids)
+	{
+		Oid				candidateOid	  = InvalidOid;
+		candidateOid = lfirst_oid(le1);
+
+		//Relation rel = (Relation)lfirst(le1);
+		Relation rel = relation_open(candidateOid, AccessShareLock);
+		if (candidateOid > 0 ) {
+			GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
+					candidateOid);
+			if(targetPolicy == NULL){
+				return GetAnalyzeVSegNumLimit();
+			}
+			if (targetPolicy->nattrs > 0) {
+				isHashRelationExist = true;
+				if(maxHashBucketNumber < targetPolicy->bucketnum){
+					maxHashBucketNumber = targetPolicy->bucketnum;
+				}
+			}
+			/*
+			 * if no hash relation, we calculate the data size of all the relations.
+			 */
+			if (!isHashRelationExist) {
+				totalDataSize += calculate_relation_size(rel);
+			}
+		}
+		relation_close(rel, AccessShareLock);
+	}
+
+	if (isHashRelationExist) {
+		vsegNumber = maxHashBucketNumber;
+	} else {
+		/*we allocate one virtual segment for each 128M data */
+		totalDataSize >>= 27;
+		vsegNumber = totalDataSize + 1;
+	}
+	Assert(vsegNumber > 0);
+	/*vsegNumber should be less than GetUtilPartitionNum*/
+	if(vsegNumber > GetQueryVsegNum()){
+		vsegNumber = GetQueryVsegNum();
+	}
+
+	return vsegNumber;
+}
+
+/*
  * This intermediate routine exists mainly to localize the effects of setjmp
  * so we don't need to plaster a lot of variables with "volatile".
  */
@@ -1809,13 +1881,21 @@ DoCopyTo(CopyState cstate)
 		 */
 		if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
 		{
-			GpPolicy *target_policy = NULL;
 			int target_segment_num = 0;
+			/*
+			 * copy hash table use table bucket number
+			 * copy random table use table size.
+			 */
+			PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
+													0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
+			Assert(pn);
+			List		*lFullRelOids = NIL;
+			lFullRelOids = all_leaf_partition_relids(pn);
+			lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id); /* root partition */
+			lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
 
-			target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
-			Assert(target_policy);
-			target_segment_num = target_policy->bucketnum;
-			pfree(target_policy);
+			target_segment_num = calculate_virtual_segment_number(lFullRelOids);
+			elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
 			CopyToDispatch(cstate);
@@ -4200,7 +4280,7 @@ CopyFrom(CopyState cstate)
 					if (cstate->oids && file_has_oids)
 						MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
 				}
-				else if ((relstorage == RELSTORAGE_PARQUET))
+				else if (relstorage == RELSTORAGE_PARQUET)
 				{
                     tuple = NULL;
 				}
@@ -4231,7 +4311,7 @@ CopyFrom(CopyState cstate)
 				{
 					HeapTuple	newtuple;
 
-					if((relstorage == RELSTORAGE_PARQUET))
+					if(relstorage == RELSTORAGE_PARQUET)
 					{
 						Assert(!tuple);
 						elog(ERROR, "triggers are not supported on tables that use column-oriented storage");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/commands/prepare.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c
index 76c04e9..013af9a 100644
--- a/src/backend/commands/prepare.c
+++ b/src/backend/commands/prepare.c
@@ -242,7 +242,7 @@ ExecuteQuery(ExecuteStmt *stmt, const char *queryString,
 									+ 255 * sizeof(pstmt->intoPolicy->attrs[0]));
 		pstmt->intoPolicy->nattrs = 0;
 		pstmt->intoPolicy->ptype = POLICYTYPE_PARTITIONED;
-		pstmt->intoPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(stmt->into->rel, GetRandomDistPartitionNum());
+		pstmt->intoPolicy->bucketnum = GetRelOpt_bucket_num_fromRangeVar(stmt->into->rel, GetDefaultPartitionNum());
 		
 		MemoryContextSwitchTo(oldContext);
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/optimizer/util/relnode.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index 9e94901..8fb676c 100644
--- a/src/backend/optimizer/util/relnode.c
+++ b/src/backend/optimizer/util/relnode.c
@@ -142,7 +142,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind)
 			{
 				rel->cdbpolicy = (GpPolicy *) palloc(sizeof(GpPolicy));
 				rel->cdbpolicy->ptype = POLICYTYPE_PARTITIONED;
-				rel->cdbpolicy->bucketnum = GetRandomDistPartitionNum();
+				rel->cdbpolicy->bucketnum = GetDefaultPartitionNum();
 				rel->cdbpolicy->nattrs = 0;
 				rel->cdbpolicy->attrs[0] = 1;
 			}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/parser/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c
index ad35484..f17c90a 100644
--- a/src/backend/parser/analyze.c
+++ b/src/backend/parser/analyze.c
@@ -2724,7 +2724,7 @@ transformETDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
 										 sizeof(p->attrs[0]));
 			p->ptype = POLICYTYPE_PARTITIONED;
 			p->nattrs = 0;
-			p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetRandomDistPartitionNum());
+			p->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
 			p->attrs[0] = 1;
 		}
 
@@ -2981,7 +2981,7 @@ transformDistributedBy(ParseState *pstate, CreateStmtContext *cxt,
 
 	if (policy->bucketnum == -1)
 	{
-		policy->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetRandomDistPartitionNum());
+		policy->bucketnum = GetRelOpt_bucket_num_fromOptions(options, GetDefaultPartitionNum());
 	}
 
 	*policyp = policy;
@@ -12187,7 +12187,7 @@ setQryDistributionPolicy(SelectStmt *stmt, Query *qry)
 	policy->nattrs = 0;
 	policy->attrs[0] = 1;
 	if(stmt->intoClause != NULL)
-		policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetRandomDistPartitionNum());
+		policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetDefaultPartitionNum());
 
 	if (stmt->distributedBy)
 	{
@@ -12241,7 +12241,7 @@ setQryDistributionPolicy(SelectStmt *stmt, Query *qry)
 		}
 		else
 		{
-			policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetRandomDistPartitionNum());
+			policy->bucketnum = GetRelOpt_bucket_num_fromOptions(stmt->intoClause->options, GetDefaultPartitionNum());
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/postmaster/identity.c
----------------------------------------------------------------------
diff --git a/src/backend/postmaster/identity.c b/src/backend/postmaster/identity.c
index b209ec5..36ea0cb 100644
--- a/src/backend/postmaster/identity.c
+++ b/src/backend/postmaster/identity.c
@@ -537,7 +537,7 @@ GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val)
 }
 
 int
-GetRandomDistPartitionNum(void)
+GetDefaultPartitionNum(void)
 {
 	return default_hash_table_bucket_number;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index a6a8bee..56909c9 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4516,7 +4516,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&default_hash_table_bucket_number,
-		6, 1, 10000, NULL, NULL
+		6, 1, 65535, NULL, NULL
 	},
 
 	{
@@ -4525,7 +4525,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_copy_from_perquery,
-		6, 1, 10000, NULL, NULL
+		6, 1, 65535, NULL, NULL
 	},
 
 	{
@@ -4534,7 +4534,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_analyze_perquery_perseg_limit,
-		4, 1, 10000, NULL, NULL
+		4, 1, 65535, NULL, NULL
 	},
 
 	{
@@ -4543,7 +4543,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&hawq_rm_nvseg_for_analyze_perquery_limit,
-		256, 1, 10000, NULL, NULL
+		256, 1, 65535, NULL, NULL
 	},
 
 	{
@@ -6388,7 +6388,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_limit,
-            512, 1, 10000, NULL, NULL
+            512, 1, 65535, NULL, NULL
     },
 
     {
@@ -6398,7 +6398,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_perseg_limit,
-            6, 1, 10000, NULL, NULL
+            6, 1, 65535, NULL, NULL
     },
 
     {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c29d6476/src/include/postmaster/identity.h
----------------------------------------------------------------------
diff --git a/src/include/postmaster/identity.h b/src/include/postmaster/identity.h
index 814935e..f43ab62 100644
--- a/src/include/postmaster/identity.h
+++ b/src/include/postmaster/identity.h
@@ -91,7 +91,7 @@ extern int	 GetCopyFromVSegNum(void);
 extern int GetRelOpt_bucket_num_fromOptions(List *options, int default_val);
 extern int GetRelOpt_bucket_num_fromRel(Relation relation, int default_val);
 extern int GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val);
-extern int GetRandomDistPartitionNum(void);
+extern int GetDefaultPartitionNum(void);
 extern int GetHashDistPartitionNum(void);
 extern int GetExternalTablePartitionNum(void);
 extern int GetAllWorkerHostNum(void);


[27/50] incubator-hawq git commit: Revert "HAWQ-531. Change GUC default value."

Posted by sh...@apache.org.
Revert "HAWQ-531. Change GUC default value."

This reverts commit 4d1a85405b87145fb931a16be9e22bd40f16eca8.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/94b7a822
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/94b7a822
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/94b7a822

Branch: refs/heads/HAWQ-459
Commit: 94b7a8225d23729d01cfc481375146d06078b601
Parents: 1897ba1
Author: hubertzhang <hz...@pivotal.io>
Authored: Thu Mar 17 15:38:46 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Thu Mar 17 15:38:46 2016 +0800

----------------------------------------------------------------------
 src/backend/utils/misc/guc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/94b7a822/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ba8e747..fa58f86 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6379,7 +6379,7 @@ static struct config_int ConfigureNamesInt[] =
                     NULL
             },
             &rm_nvseg_perquery_limit,
-            512, 1, 65535, NULL, NULL
+            1000, 1, 65535, NULL, NULL
     },
 
     {
@@ -6535,7 +6535,7 @@ static struct config_int ConfigureNamesInt[] =
 			NULL
 		},
 		&rm_rejectrequest_nseg_limit,
-		4, 0, 65535, NULL, NULL
+		2, 0, 65535, NULL, NULL
 	},
 
 	{


[11/50] incubator-hawq git commit: HAWQ-535. hawqextract column context does not exist error

Posted by sh...@apache.org.
HAWQ-535. hawqextract column context does not exist error


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5152fb0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5152fb0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5152fb0b

Branch: refs/heads/HAWQ-459
Commit: 5152fb0b02b6bf469d48c039d17a4780075ad578
Parents: f34f65d
Author: dlynch <dl...@gopivotal.com>
Authored: Fri Jan 29 18:13:14 2016 -0600
Committer: rlei <rl...@pivotal.io>
Committed: Tue Mar 15 10:46:31 2016 +0800

----------------------------------------------------------------------
 tools/bin/hawqextract | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5152fb0b/tools/bin/hawqextract
----------------------------------------------------------------------
diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract
index be08117..dea12fb 100755
--- a/tools/bin/hawqextract
+++ b/tools/bin/hawqextract
@@ -152,12 +152,12 @@ class GpMetadataAccessor:
 
         Example:
         >>> accessor.get_aoseg_files(35709)
-        >>> [{'content':'0', 'fileno':'1', 'filesize':'320'},
-        ...  {'content':'1', 'fileno':'1', 'filesize':'880'},
-        ...  {'content':'1', 'fileno':'2', 'filesize':'160'}]
+        >>> [{'fileno':'1', 'filesize':'320'},
+        ...  {'fileno':'2', 'filesize':'880'},
+        ...  {'fileno':'3', 'filesize':'160'}]
         '''
         qry = """
-        SELECT content, segno as fileno, eof as filesize 
+        SELECT segno as fileno, eof as filesize 
         FROM pg_aoseg.pg_aoseg_%d
         ORDER by fileno;
         """ % oid
@@ -170,14 +170,14 @@ class GpMetadataAccessor:
 
         Example:
         >>> accessor.get_paqseg_files(35709)
-        >>> [{'content':'0', 'fileno':'1', 'filesize':'320'},
-        ...  {'content':'1', 'fileno':'1', 'filesize':'880'},
-        ...  {'content':'1', 'fileno':'2', 'filesize':'160'}]
+        >>> [{'fileno':'1', 'filesize':'320'},
+        ...  {'fileno':'2', 'filesize':'880'},
+        ...  {'fileno':'3', 'filesize':'160'}]
         '''
         qry = """
-        SELECT content, segno as fileno, eof as filesize 
+        SELECT segno as fileno, eof as filesize 
         FROM pg_aoseg.pg_paqseg_%d
-        WHERE content >= 0 ORDER by content, fileno;
+        ORDER by fileno;
         """ % oid
         return self.exec_query(qry)
 
@@ -306,17 +306,17 @@ def extract_metadata(conn, tbname):
 
         Example:
         >>> segment_localtions
-        >>> ['hdfs://127.0.0.1:9000/gpseg0', 'hdfs://127.0.0.1:9000/gpseg1']
+        >>> ['hdfs://127.0.0.1:9000/hawq_default', 'hdfs://127.0.0.1:9000/hawq_default']
         >>> tablespace_oid, database_oid, relfilenode, oid
         >>> (16385, 35469, 35470, 35488)
         >>> accessor.get_aoseg_files(35488)
-        >>> [{'content': '0', 'fileno': '1', 'filesize': '180'},
-        ...  {'content': '0', 'fileno': '2', 'filesize': '150'},
-        ...  {'content': '1', 'fileno': '1', 'filesize': '320'}]
+        >>> [{'fileno': '1', 'filesize': '180'},
+        ...  {'fileno': '2', 'filesize': '150'},
+        ...  {'fileno': '3', 'filesize': '320'}]
         >>> get_ao_table_files(35488, 35470)
-        >>> [{'path' :'/gpseg0/16385/35469/35470.1', 'size': 180},
-        ...  {'path' :'/gpseg0/16385/35469/35470.2', 'size': 150},
-        ...  {'path' :'/gpseg1/16385/35469/35470.1', 'size': 320}]
+        >>> [{'path' :'/hawq_default/16385/35469/1', 'size': 180},
+        ...  {'path' :'/hawq_default/16385/35469/2', 'size': 150},
+        ...  {'path' :'/hawq_default/16385/35469/3', 'size': 320}]
         '''
         files = []
         for f in accessor.get_aoseg_files(oid):


[03/50] incubator-hawq git commit: Support --enable-orca [#105338218]

Posted by sh...@apache.org.
Support --enable-orca [#105338218]

fix gpopt interface change
fix boost_thread to use boost_thread_mt on OSX
fix version detection using ivy.xml


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a48600a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a48600a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a48600a1

Branch: refs/heads/HAWQ-459
Commit: a48600a125eed2a3da00133d6f5f8dad22d4a6a4
Parents: 29a9beb
Author: Jacob Frank <jf...@pivotal.io>
Authored: Tue Mar 8 17:49:07 2016 -0800
Committer: C.J. Jameson <cj...@pivotal.io>
Committed: Mon Mar 14 12:24:56 2016 -0700

----------------------------------------------------------------------
 configure                          | 34 --------------------------------
 configure.in                       | 35 ---------------------------------
 src/backend/Makefile               | 19 +++++++++---------
 src/backend/gpopt/Makefile         | 25 ++++++++++-------------
 src/backend/gpopt/gpopt.mk         | 24 ++++++++++++----------
 src/bin/gpcheckhdfs/Makefile       |  2 +-
 src/bin/gpfilesystem/hdfs/Makefile |  2 +-
 7 files changed, 36 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5135536..5669406 100755
--- a/configure
+++ b/configure
@@ -731,11 +731,6 @@ with_perl
 with_tcl
 enable_thread_safety
 INCLUDES
-ORCA_DEPENDS_OPTIMIZER_VER
-ORCA_DEPENDS_LIBGPOS_VER
-ORCA_DEPENDS_XERCES_VER
-ORCA_DEPENDS_OBJDIR_DEFAULT
-ORCA_DEPENDS_DIR_INTER
 enable_orca
 enable_email
 enable_gphdfs
@@ -5522,35 +5517,6 @@ fi
 
 
 
-if test "$enable_orca" = yes ; then
-
-	if test -z "$ORCA_DEPENDS_DIR" ; then
-		as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
-		as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$XERCES_VER" ; then
-		as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$LIBGPOS_VER" ; then
-		as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$OPTIMIZER_VER" ; then
-		as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
-	fi
-
-    ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
-    ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
-    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
-    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
-    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
-fi
-
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index cda7bad..489187e 100644
--- a/configure.in
+++ b/configure.in
@@ -639,41 +639,6 @@ PGAC_ARG_BOOL(enable, orca, no, [  --enable-orca        enable Pivotal Query Opt
                          [Define to 1 to build with Pivotal Query Optimizer. (--enable-orca)])])
 AC_SUBST(enable_orca)
 
-if test "$enable_orca" = yes ; then
-
-	if test -z "$ORCA_DEPENDS_DIR" ; then
-		AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
-	fi
-
-	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
-		AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
-	fi
-	
-	if test -z "$XERCES_VER" ; then
-		AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
-	fi
-	
-	if test -z "$LIBGPOS_VER" ; then
-		AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
-	fi
-	
-	if test -z "$OPTIMIZER_VER" ; then
-		AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
-	fi
-
-    ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
-    ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
-    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
-    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
-    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
-fi
-
-AC_SUBST(ORCA_DEPENDS_DIR_INTER)
-AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
-AC_SUBST(ORCA_DEPENDS_XERCES_VER)
-AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
-AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
-
 #
 # Include directories
 #

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 8460b76..f462510 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,15 +52,15 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
 
 # The backend doesn't need everything that's in LIBS, however
 LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
 
 # adding orca libraries
 ifeq ($(enable_orca),yes)
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpos/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libnaucrates/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost
-LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpopt/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt
+LIBS := $(LIBS) -lgpos
+LIBS := $(LIBS) -lxerces-c
+LIBS := $(LIBS) -lnaucrates
+LIBS := $(LIBS) -lgpdbcost
+LIBS := $(LIBS) -lgpopt
 LIBS := $(LIBS) -L$(top_builddir)/src/backend/gpopt -ldxltranslators
 endif
 
@@ -213,9 +213,6 @@ ifeq ($(MAKE_DLL), true)
 	$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
 endif
 endif
-ifeq ($(enable_orca),yes)
-	$(MAKE) -C gpopt install
-endif
 	$(MAKE) -C catalog install-data
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -228,6 +225,10 @@ endif
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
+ifeq ($(enable_orca), yes)
+	$(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
+endif
+
 
 install-bin: postgres $(POSTGRES_IMP) installdirs
 	$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/gpopt/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/Makefile b/src/backend/gpopt/Makefile
index b587ffd..a7ffa87 100644
--- a/src/backend/gpopt/Makefile
+++ b/src/backend/gpopt/Makefile
@@ -39,29 +39,24 @@ else
 	GPOPT_UDF_FLAGS := -shared
 endif
 
-LIBGPOS_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpos
-LIBNAUCRATES_DIR = $(ORCA_DEPENDS_DIR_INTER)/libnaucrates
-LIBGPDBCOST_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost
-LIBGPOPT_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpopt
-
 include $(top_srcdir)/src/backend/common.mk
 
 libdxltranslators.$(LDSFX): $(SUBDIROBJS) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(LDLIBFLAGS) \
-	  -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
-	  -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
-	  -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
-	  -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
-	  -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt -lpthread -o $@ \
+	  -lgpos \
+	  -lxerces-c \
+	  -lnaucrates \
+	  -lgpdbcost \
+	  -lgpopt -lpthread -o $@ \
 	  $(SUBDIROBJS) $(OBJS)
 
 libgpoptudf.$(LDSFX): $(SUBDIROBJS)
 	$(CXX) $(CXXFLAGS) $(GPOPT_UDF_FLAGS) \
-	  -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
-	  -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
-	  -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
-	  -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
-	  -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt \
+	  -lxerces-c \
+	  -lgpos \
+	  -lnaucrates \
+	  -lgpdbcost \
+	  -lgpopt \
 	  -o $@ ./utils/funcs.o
 
 install:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 6c77d2f..0099667 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,6 +25,8 @@
 
 UNAME = $(shell uname)
 UNAME_P = $(shell uname -p)
+UNAME_M = $(shell uname -m)
+
 ARCH_OS = GPOS_$(UNAME)
 ARCH_CPU = GPOS_$(UNAME_P)
 
@@ -41,8 +43,9 @@ ifeq "$(BLD_TYPE)" "opt"
 	GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
 endif
 
-ARCH_BIT = GPOS_64BIT
-ifeq (Darwin, $(UNAME))
+ifeq (x86_64, $(UNAME_M))
+	ARCH_BIT = GPOS_64BIT
+else
 	ARCH_BIT = GPOS_32BIT
 endif
 
@@ -52,14 +55,15 @@ else
 	ARCH_FLAGS = -m64
 endif
 
+GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
+
+XERCES_VER  = $(shell grep "\"xerces-c\""   $(GREP_SED_VAR))
+LIBGPOS_VER = $(shell grep "\"libgpos\""    $(GREP_SED_VAR))
+OPTIMIZER_VER = $(shell grep "\"optimizer\""  $(GREP_SED_VAR))
+
 BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
 override CPPFLAGS := -fPIC $(CPPFLAGS)
 override CPPFLAGS := $(BLD_FLAGS)  $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libnaucrates/include $(CPPFLAGS)
-override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/include $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index cd81902..0b88600 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
 
 OBJS=gpcheckhdfs.o 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a48600a1/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index b0fb359..2399789 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS       = gpfshdfs.o
 
 PG_CPPFLAGS = -I$(libpq_srcdir)
 PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3
 
 ifdef USE_PGXS
 PGXS := $(shell pg_config --pgxs)


[39/50] incubator-hawq git commit: HAWQ-558. The query running on random table does not push resource manager to request resource from YARN as maimum possible resource consumption

Posted by sh...@apache.org.
HAWQ-558. The query running on random table does not push resource manager to request resource from YARN as maimum possible resource consumption


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b3a35218
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b3a35218
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b3a35218

Branch: refs/heads/HAWQ-459
Commit: b3a352186cbe4ebadc5b76121cbefde474fc3a53
Parents: e82ac3f
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 14:13:34 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 14:13:34 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcemanager.c | 83 +++++++++++++---------
 1 file changed, 51 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b3a35218/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 5a01316..c585d40 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -78,6 +78,7 @@ void completeAllocRequestToBroker(int32_t 	 *reqmem,
 								  int32_t 	 *reqcore,
 								  List 		**preferred);
 void processResourceBrokerTasks(void);
+void generateResourceRequestToResourceBroker(void);
 void cleanupAllGRMContainers(void);
 bool cleanedAllGRMContainers(void);
 
@@ -610,12 +611,15 @@ int MainHandlerLoop(void)
 		/* STEP 5. Handle all submitted requests through socket clients. */
 		processSubmittedRequests();
 
-        /* STEP 6. Check timeout resource allocation and timeout queuing requests. */
+		/* STEP 6. Generate possible resource request to resource broker. */
+		generateResourceRequestToResourceBroker();
+
+        /* STEP 7. Check timeout resource allocation and timeout queuing requests. */
         timeoutDeadResourceAllocation();
         timeoutQueuedRequest();
 
 		/*
-		 * STEP 7. Check the status of all segment nodes, mark down if hasn't got
+		 * STEP 8. Check the status of all segment nodes, mark down if hasn't got
 		 * 		   IMAlive message for a pre-defined period.
 		 */
         uint64_t curtime = gettime_microsec();
@@ -628,12 +632,13 @@ int MainHandlerLoop(void)
 			PRESPOOL->LastCheckTime = curtime;
 		}
 
-		/* STEP 8. Move all accepted GRM containers into resource pool. */
+
+		/* STEP 9. Move all accepted GRM containers into resource pool. */
 		moveAllAcceptedGRMContainersToResPool();
 
 		/*
-		 * Check if should pause dispatching resource to queries to collect
-		 * resource back in order to return GRM containers.
+		 * STEP 10. Check if should pause dispatching resource to queries to
+		 * collect resource back in order to return GRM containers.
 		 */
 		if ( DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
 			 PRESPOOL->AddPendingContainerCount == 0 &&
@@ -654,7 +659,8 @@ int MainHandlerLoop(void)
 			setForcedReturnGRMContainerCount();
 		}
 
-		/* STEP 9. Dispatch resource to queries and send the messages out.*/
+		/* STEP 11. Dispatch resource to queries and send the messages out.*/
+
         if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 &&
 			 PQUEMGR->toRunQueryDispatch &&
 			 PQUEMGR->ForcedReturnGRMContainerCount == 0 &&
@@ -675,23 +681,23 @@ int MainHandlerLoop(void)
 						 PQUEMGR->ForcedReturnGRMContainerCount);
         }
 
-        /* STEP 10. Generate output content to client connections. */
+        /* STEP 12. Generate output content to client connections. */
         sendResponseToClients();
 
         /*
-         * STEP 11. Return containers to global resource manager if there some
+         * STEP 13. Return containers to global resource manager if there some
          * 			some idle resource.
          */
         timeoutIdleGRMResourceToRB();
 
-		/* STEP 12. Notify segments to increase resource quota. */
+		/* STEP 14. Notify segments to increase resource quota. */
 		notifyToBeAcceptedGRMContainersToRMSEG();
 
-        /* STEP 13. Notify segments to decrease resource. */
+        /* STEP 15. Notify segments to decrease resource. */
         notifyToBeKickedGRMContainersToRMSEG();
 
         /*
-         * STEP 14. Check slaves file if the content is not checked or is
+         * STEP 16. Check slaves file if the content is not checked or is
          * 			updated.
          */
         checkSlavesFile();
@@ -2881,26 +2887,7 @@ void processResourceBrokerTasks(void)
 			}
 		}
 
-		/* STEP 3. Generate GRM container allocation request. */
-		curtime = gettime_microsec();
-
-        if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 )
-        {
-        	refreshMemoryCoreRatioLevelUsage(curtime);
-        	if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
-        	{
-        		PRESPOOL->LastResAcqTime = curtime;
-				res = generateAllocRequestToBroker();
-				if ( res != FUNC_RETURN_OK )
-				{
-					elog(WARNING, "Resource manager fails to allocate container "
-								  "from global resource manager.");
-					goto exit;
-				}
-        	}
-        }
-
-        /* STEP 4. Return kicked GRM containers. */
+        /* STEP 3. Return kicked GRM containers. */
         curtime = gettime_microsec();
 
     	if ( !PRESPOOL->pausePhase[QUOTA_PHASE_KICKED_TO_RETURN] )
@@ -2918,7 +2905,7 @@ void processResourceBrokerTasks(void)
 			elog(LOG, "Paused returning GRM containers kicked to GRM.");
     	}
 	    /*
-	     * STEP 5. Handle resource broker input as new allocated resource or
+	     * STEP 4. Handle resource broker input as new allocated resource or
 		 * 		   cluster report, container report etc. The allocated resource
 		 * 		   from resource broker will be added to the resource queue
 		 * 		   manager and resource pool.
@@ -2940,6 +2927,38 @@ exit:
 	RB_handleError(res);
 }
 
+void generateResourceRequestToResourceBroker(void)
+{
+	uint64_t curtime = 0;
+	int		 res	 = FUNC_RETURN_OK;
+
+	if ( !isCleanGRMResourceStatus() )
+	{
+
+		curtime = gettime_microsec();
+
+        if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 )
+        {
+        	refreshMemoryCoreRatioLevelUsage(curtime);
+        	if ( curtime - PRESPOOL->LastResAcqTime > 1000000LL)
+        	{
+        		PRESPOOL->LastResAcqTime = curtime;
+				res = generateAllocRequestToBroker();
+				if ( res != FUNC_RETURN_OK )
+				{
+					elog(WARNING, "Resource manager fails to allocate container "
+								  "from global resource manager.");
+					goto exit;
+				}
+        	}
+        }
+	}
+
+exit:
+	RB_handleError(res);
+
+}
+
 void cleanupAllGRMContainers(void)
 {
 	dropAllResPoolGRMContainersToToBeKicked();


[13/50] incubator-hawq git commit: HAWQ-473. fix coverity errors introduced by new codes

Posted by sh...@apache.org.
HAWQ-473. fix coverity errors introduced by new codes


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c72d0662
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c72d0662
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c72d0662

Branch: refs/heads/HAWQ-459
Commit: c72d06629f28b00b430db0bfcc79422cfd25f814
Parents: 99a307b
Author: Wen Lin <wl...@pivotal.io>
Authored: Tue Mar 15 14:55:37 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Mar 15 14:55:37 2016 +0800

----------------------------------------------------------------------
 .../communication/rmcomm_RM2RMSEG.c             | 16 +++++---------
 src/backend/resourcemanager/requesthandler.c    |  8 +++----
 .../resourcebroker/resourcebroker_LIBYARN.c     |  8 +++----
 src/backend/resourcemanager/resourcemanager.c   | 12 +++++-----
 src/backend/resourcemanager/resourcepool.c      | 23 +++++++-------------
 5 files changed, 25 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index ca332c9..7baab7d 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -243,11 +243,9 @@ void receivedRUAliveResponse(AsyncCommMessageHandlerContext  context,
 				add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
 										GET_SEGRESOURCE_HOSTNAME(segres),
 										description->Str);
-				if (description != NULL)
-				{
-					freeSimpleStringContent(description);
-					rm_pfree(PCONTEXT, description);
-				}
+
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
 			}
 			/* Set the host down. */
 			elog(WARNING, "Resource manager sets host %s from up to down "
@@ -304,11 +302,9 @@ void sentRUAliveError(AsyncCommMessageHandlerContext context)
 			add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
 									GET_SEGRESOURCE_HOSTNAME(segres),
 									description->Str);
-			if (description != NULL)
-			{
-				freeSimpleStringContent(description);
-				rm_pfree(PCONTEXT, description);
-			}
+
+			freeSimpleStringContent(description);
+			rm_pfree(PCONTEXT, description);
 		}
 		/* Set the host down. */
 		elog(LOG, "Resource manager sets host %s from up to down "

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index a06d169..694c9cd 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -1014,11 +1014,9 @@ bool handleRMRequestSegmentIsDown(void **arg)
 						add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
 												hostname,
 												description->Str);
-						if (description != NULL)
-						{
-							freeSimpleStringContent(description);
-							rm_pfree(PCONTEXT, description);
-						}
+
+						freeSimpleStringContent(description);
+						rm_pfree(PCONTEXT, description);
 					}
 
 					/* Set the host down. */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index d446a6d..cdbe0d5 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -732,11 +732,9 @@ int handleRB2RM_ClusterReport(void)
 						GET_SEGRESOURCE_HOSTNAME(segres),
 						SEGMENT_STATUS_DOWN,
 						(description->Len > 0)?description->Str:"");
-			if (description != NULL)
-			{
-				freeSimpleStringContent(description);
-				rm_pfree(PCONTEXT, description);
-			}
+
+			freeSimpleStringContent(description);
+			rm_pfree(PCONTEXT, description);
 		}
 	}
 	freePAIRRefList(&(PRESPOOL->Segments), &allsegres);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 138c5a0..cb1eecc 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -2557,9 +2557,9 @@ void updateStatusOfAllNodes()
 	for(uint32_t idx = 0; idx < PRESPOOL->SegmentIDCounter; idx++)
 	{
 		node = getSegResource(idx);
+		Assert(node != NULL);
 		uint8_t oldStatus = node->Stat->FTSAvailable;
-		if (node != NULL &&
-			 (curtime - node->LastUpdateTime >
+		if ( (curtime - node->LastUpdateTime >
 			 1000000LL * rm_segment_heartbeat_timeout) &&
 			 (node->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0)
 		{
@@ -2591,11 +2591,9 @@ void updateStatusOfAllNodes()
 				add_segment_history_row(idx + REGISTRATION_ORDER_OFFSET,
 										GET_SEGRESOURCE_HOSTNAME(node),
 										(description->Len > 0)?description->Str:"");
-				if (description != NULL)
-				{
-					freeSimpleStringContent(description);
-					rm_pfree(PCONTEXT, description);
-				}
+
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
 			}
 
 			elog(WARNING, "Resource manager sets host %s heartbeat timeout.",

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c72d0662/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index d3a01a4..ff5f34f 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -992,11 +992,8 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 									IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
 										SEG_STATUS_DESCRIPTION_UP:description->Str);
 
-			if (description != NULL)
-			{
-				freeSimpleStringContent(description);
-				rm_pfree(PCONTEXT, description);
-			}
+			freeSimpleStringContent(description);
+			rm_pfree(PCONTEXT, description);
 		}
 
 		if (segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
@@ -1243,11 +1240,9 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 										GET_SEGRESOURCE_HOSTNAME(segresource),
 										IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
 											SEG_STATUS_DESCRIPTION_UP:description->Str);
-				if (description != NULL)
-				{
-					freeSimpleStringContent(description);
-					rm_pfree(PCONTEXT, description);
-				}
+
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
 			}
 		}
 
@@ -1538,11 +1533,9 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 					IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
 						SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
 					(description->Len > 0)?description->Str:"");
-		if (description != NULL)
-		{
-			freeSimpleStringContent(description);
-			rm_pfree(PCONTEXT, description);
-		}
+
+		freeSimpleStringContent(description);
+		rm_pfree(PCONTEXT, description);
 	}
 
 	int32_t curratio = 0;


[43/50] incubator-hawq git commit: HAWQ-502 Enable ORCA [#105338218]

Posted by sh...@apache.org.
HAWQ-502 Enable ORCA [#105338218]

This adds `--enable-orca` in autoconf.
This commit is the first step in fixing apache/incuabor-hawq#420 .


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c67a1a43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c67a1a43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c67a1a43

Branch: refs/heads/HAWQ-459
Commit: c67a1a43c7159eda9d5d6f70110d6db9425110f0
Parents: 1d9eb5e
Author: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Authored: Fri Mar 18 17:01:37 2016 -0700
Committer: Jacob Frank, Jesse Zhang, Omer Arap and Xin Zhang <jz...@pivotal.io>
Committed: Fri Mar 18 17:01:37 2016 -0700

----------------------------------------------------------------------
 configure                  | 27 ---------------------------
 configure.in               | 27 ---------------------------
 src/backend/Makefile       |  6 +++---
 src/backend/gpopt/gpopt.mk | 18 +++++++++++++-----
 4 files changed, 16 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5135536..9dffef3 100755
--- a/configure
+++ b/configure
@@ -731,9 +731,6 @@ with_perl
 with_tcl
 enable_thread_safety
 INCLUDES
-ORCA_DEPENDS_OPTIMIZER_VER
-ORCA_DEPENDS_LIBGPOS_VER
-ORCA_DEPENDS_XERCES_VER
 ORCA_DEPENDS_OBJDIR_DEFAULT
 ORCA_DEPENDS_DIR_INTER
 enable_orca
@@ -5523,32 +5520,8 @@ fi
 
 
 if test "$enable_orca" = yes ; then
-
-	if test -z "$ORCA_DEPENDS_DIR" ; then
-		as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
-		as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$XERCES_VER" ; then
-		as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$LIBGPOS_VER" ; then
-		as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
-	fi
-
-	if test -z "$OPTIMIZER_VER" ; then
-		as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
-	fi
-
     ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
     ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
-    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
-    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
-    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
 fi
 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index cda7bad..13260ef 100644
--- a/configure.in
+++ b/configure.in
@@ -640,39 +640,12 @@ PGAC_ARG_BOOL(enable, orca, no, [  --enable-orca        enable Pivotal Query Opt
 AC_SUBST(enable_orca)
 
 if test "$enable_orca" = yes ; then
-
-	if test -z "$ORCA_DEPENDS_DIR" ; then
-		AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
-	fi
-
-	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
-		AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
-	fi
-	
-	if test -z "$XERCES_VER" ; then
-		AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
-	fi
-	
-	if test -z "$LIBGPOS_VER" ; then
-		AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
-	fi
-	
-	if test -z "$OPTIMIZER_VER" ; then
-		AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
-	fi
-
     ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
     ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
-    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
-    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
-    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
 fi
 
 AC_SUBST(ORCA_DEPENDS_DIR_INTER)
 AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
-AC_SUBST(ORCA_DEPENDS_XERCES_VER)
-AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
-AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
 
 #
 # Include directories

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 8460b76..18ed91a 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -213,9 +213,6 @@ ifeq ($(MAKE_DLL), true)
 	$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
 endif
 endif
-ifeq ($(enable_orca),yes)
-	$(MAKE) -C gpopt install
-endif
 	$(MAKE) -C catalog install-data
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -228,6 +225,9 @@ endif
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
+ifeq ($(enable_orca), yes)
+	$(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
+endif
 
 install-bin: postgres $(POSTGRES_IMP) installdirs
 	$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c67a1a43/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 6c77d2f..1fa7d81 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,6 +25,7 @@
 
 UNAME = $(shell uname)
 UNAME_P = $(shell uname -p)
+UNAME_M = $(shell uname -m)
 ARCH_OS = GPOS_$(UNAME)
 ARCH_CPU = GPOS_$(UNAME_P)
 
@@ -41,8 +42,9 @@ ifeq "$(BLD_TYPE)" "opt"
 	GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
 endif
 
-ARCH_BIT = GPOS_64BIT
-ifeq (Darwin, $(UNAME))
+ifeq (x86_64, $(UNAME_M))
+	ARCH_BIT = GPOS_64BIT
+else
 	ARCH_BIT = GPOS_32BIT
 endif
 
@@ -52,12 +54,18 @@ else
 	ARCH_FLAGS = -m64
 endif
 
+GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
+
+XERCES_VER  = $(shell grep "\"xerces-c\""   $(GREP_SED_VAR))
+LIBGPOS_VER = $(shell grep "\"libgpos\""    $(GREP_SED_VAR))
+OPTIMIZER_VER = $(shell grep "\"optimizer\""  $(GREP_SED_VAR))
+
 BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
 override CPPFLAGS := -fPIC $(CPPFLAGS)
 override CPPFLAGS := $(BLD_FLAGS)  $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)
 override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
 override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
 override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)


[37/50] incubator-hawq git commit: HAWQ-556. Resource manager uses unadjusted water level guc for allocating GRM resource containers

Posted by sh...@apache.org.
HAWQ-556. Resource manager uses unadjusted water level guc for allocating GRM resource containers


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f38ad114
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f38ad114
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f38ad114

Branch: refs/heads/HAWQ-459
Commit: f38ad11472cd3c8084ac156123583239e192be5b
Parents: 17af44c
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:49:09 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:49:09 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcemanager.c | 2 +-
 src/backend/resourcemanager/resqueuemanager.c | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f38ad114/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 21bf0d1..5a01316 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -2065,7 +2065,7 @@ int generateAllocRequestToBroker(void)
 	}
 
 	/* Decide water level of resource. */
-	int wlevel = hasWorkload ? rm_min_resource_perseg : 0;
+	int wlevel = hasWorkload ? PQUEMGR->ActualMinGRMContainerPerSeg : 0;
 	switch( DRMGlobalInstance->ImpType )
 	{
 	case YARN_LIBYARN:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f38ad114/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index cbf824e..538c2ad 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -5213,8 +5213,10 @@ void getIdleResourceRequest(int32_t *mem, double *core)
 	Assert(PRESPOOL->ClusterMemoryCoreRatio > 0);
 	*mem  = PRESPOOL->ClusterMemoryCoreRatio *
 			PRESPOOL->AvailNodeCount *
-			rm_min_resource_perseg;
-	*core = 1.0 * PRESPOOL->AvailNodeCount * rm_min_resource_perseg;
+			PQUEMGR->ActualMinGRMContainerPerSeg;
+	*core = 1.0 *
+			PRESPOOL->AvailNodeCount *
+			PQUEMGR->ActualMinGRMContainerPerSeg;
 }
 
 void setForcedReturnGRMContainerCount(void)


[15/50] incubator-hawq git commit: HAWQ-542. Passing empty string to attribute vseg_resource_quota makes default value set instead of reporting error

Posted by sh...@apache.org.
HAWQ-542. Passing empty string to attribute vseg_resource_quota makes default value set instead of reporting error


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/31eb645b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/31eb645b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/31eb645b

Branch: refs/heads/HAWQ-459
Commit: 31eb645b44bbd8ec453dbcb33a491c16f3f69178
Parents: ca2a770
Author: YI JIN <yj...@pivotal.io>
Authored: Wed Mar 16 13:37:55 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Wed Mar 16 13:37:55 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resqueuemanager.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/31eb645b/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 7bc0e7b..cbf824e 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -346,6 +346,17 @@ int shallowparseResourceQueueWithAttributes(List 	*rawattr,
 		case RSQ_DDL_ATTR_NVSEG_UPPER_LIMIT_PERSEG:
 		case RSQ_DDL_ATTR_NVSEG_LOWER_LIMIT_PERSEG:
 		{
+
+			/* The empty value is not allowed. */
+			if ( SimpleStringEmpty(&(property->Val)) )
+			{
+				snprintf(errorbuf, errorbufsize,
+						 "the value of %s cannot be empty",
+						 RSQDDLAttrNames[attrindex]);
+				elog(WARNING, "Resource manager failed parsing attribute, %s",
+							  errorbuf);
+				return RMDDL_WRONG_ATTRVALUE;
+			}
 			/*
 			 * Build property.
 			 *


[10/50] incubator-hawq git commit: HAWQ-534. A query resource quota can be calculated to from 0 to 0

Posted by sh...@apache.org.
HAWQ-534. A query resource quota can be calculated to from 0 to 0


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f34f65de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f34f65de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f34f65de

Branch: refs/heads/HAWQ-459
Commit: f34f65de1a2db5de7d1768c25f8eb6fa15adf578
Parents: a7ad375
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Mar 15 13:36:58 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Mar 15 13:36:58 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resqueuemanager.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f34f65de/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 9af0d7c..7bc0e7b 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -3868,7 +3868,7 @@ int computeQueryQuota_EVEN(DynResourceQueueTrack	track,
 	*segnummin = reservsegnum;
 	*segnummin = *segnummin > *segnum ? *segnum : *segnummin;
 
-	Assert( *segnummin > 0 && *segnummin <= *segnum );
+	Assert( *segnummin >= 0 && *segnummin <= *segnum );
 	return FUNC_RETURN_OK;
 }
 


[29/50] incubator-hawq git commit: HAWQ-550. Auto clean up records in gp_configuration_history longer than a GUC value, default is 365 days

Posted by sh...@apache.org.
HAWQ-550. Auto clean up records in gp_configuration_history longer than a GUC value, default is 365 days


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/64663eff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/64663eff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/64663eff

Branch: refs/heads/HAWQ-459
Commit: 64663eff91a03c2c95bb1f4f145bdf502253a805
Parents: dfc6731
Author: Wen Lin <wl...@pivotal.io>
Authored: Fri Mar 18 09:37:38 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Fri Mar 18 09:37:38 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbvars.c                       |  1 +
 .../resourcemanager/include/resourcepool.h      |  3 +
 .../resourcemanager/requesthandler_RMSEG.c      |  5 +-
 src/backend/resourcemanager/resourcemanager.c   |  2 +
 src/backend/resourcemanager/resourcepool.c      | 74 +++++++++++++++++++-
 src/backend/utils/misc/guc.c                    |  9 +++
 src/include/cdb/cdbvars.h                       |  1 +
 7 files changed, 92 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index fbb4bc0..b4a9f3d 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -297,6 +297,7 @@ int    seg_addr_port;
 char  *dfs_url;
 char  *master_directory;
 char  *seg_directory;
+int    segment_history_keep_period;
 
 /* HAWQ 2.0 resource manager GUCs */
 int    rm_master_port;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index 5feedd3..f965c43 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -670,6 +670,9 @@ void adjustMemoryCoreValue(uint32_t *memorymb, uint32_t *core);
 /* Clean up gp_segment_configuration */
 void cleanup_segment_config(void);
 
+/* Clean up gp_configuration_history */
+void cleanup_segment_config_history(void);
+
 #define SEG_STATUS_DESCRIPTION_UP "segment is up"
 /* update a segment's status in gp_segment_configuration table */
 void update_segment_status(int32_t id, char status, char* description);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/requesthandler_RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler_RMSEG.c b/src/backend/resourcemanager/requesthandler_RMSEG.c
index 90946a0..12bd8ac 100644
--- a/src/backend/resourcemanager/requesthandler_RMSEG.c
+++ b/src/backend/resourcemanager/requesthandler_RMSEG.c
@@ -307,7 +307,8 @@ bool handleRMSEGRequestRUAlive(void **arg)
 		{
 			if (retry == 0)
 			{
-				elog(LOG, "Segment's postmaster is down, PQconnectdb result : %d, %s",
+				elog(LOG, "Segment receives RUAlive from master "
+						  "and postmaster is down, PQconnectdb result : %d, %s",
 						  libpqres,
 						  PQerrorMessage(conn));
 				/* Don't send IMAlive anymore */
@@ -322,7 +323,7 @@ bool handleRMSEGRequestRUAlive(void **arg)
 		}
 		else
 		{
-			elog(DEBUG3, "Segment's postmaster is healthy.");
+			elog(LOG, "Segment receives RUAlive from master and postmaster is healthy.");
 			break;
 		}
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index cb1eecc..45a669e 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -479,6 +479,8 @@ int ResManagerMainServer2ndPhase(void)
 	/* Clean up gp_segment_configuration table. */
 	cleanup_segment_config();
 
+	cleanup_segment_config_history();
+
 	/*
 	 * register itself into gp_segment_configuration table
 	 * master internal id is 0, segment id starts from 1

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index ff5f34f..ca275f5 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -28,6 +28,7 @@
 #include "gp-libpq-int.h"
 #include "utils/builtins.h"
 #include "catalog/pg_proc.h"
+#include "cdb/cdbvars.h"
 
 void addSegResourceAvailIndex(SegResource segres);
 void addSegResourceAllocIndex(SegResource segres);
@@ -471,6 +472,77 @@ cleanup:
 }
 
 /*
+ * Remove records in gp_configuration_history that are longer than
+ * a period defined in GUC, the default values is 365 days.
+ */
+void cleanup_segment_config_history(){
+	int	libpqres = CONNECTION_OK;
+	PGconn *conn = NULL;
+	char conninfo[512];
+	PQExpBuffer sql = NULL;
+	PGresult* result = NULL;
+
+	sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
+			"dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
+	conn = PQconnectdb(conninfo);
+	if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
+	{
+		elog(WARNING, "Fail to connect database when cleanup "
+				      "segment history catalog table, error code: %d, %s",
+				      libpqres,
+				      PQerrorMessage(conn));
+		PQfinish(conn);
+		return;
+	}
+
+	result = PQexec(conn, "BEGIN");
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+				      "segment history catalog table, reason : %s",
+				      "BEGIN",
+				      PQresultErrorMessage(result));
+		goto cleanup;
+	}
+	PQclear(result);
+
+	sql = createPQExpBuffer();
+	appendPQExpBuffer(sql,"DELETE FROM gp_configuration_history WHERE "
+						  "current_timestamp - time > interval '%d days'",
+						  segment_history_keep_period);
+	result = PQexec(conn, sql->data);
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+				      "segment history catalog table, reason : %s",
+				      sql->data,
+				      PQresultErrorMessage(result));
+		goto cleanup;
+	}
+	PQclear(result);
+
+	result = PQexec(conn, "COMMIT");
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+				      "segment history catalog table, reason : %s",
+				      "COMMIT",
+				      PQresultErrorMessage(result));
+		goto cleanup;
+	}
+
+	elog(LOG, "Cleanup segment configuration history catalog table successfully, "
+			  "keep period: recent %d days.", segment_history_keep_period);
+
+cleanup:
+	if(sql)
+		destroyPQExpBuffer(sql);
+	if(result)
+		PQclear(result);
+	PQfinish(conn);
+}
+
+/*
  * Remove all entries in gp_configuration_history.
  *
  * gp_remove_segment_history()
@@ -500,7 +572,7 @@ gp_remove_segment_history(PG_FUNCTION_ARGS)
 	if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
 	{
 		elog(WARNING, "Fail to connect database when cleanup "
-					  "segment configuration catalog table, error code: %d, %s",
+					  "segment history catalog table, error code: %d, %s",
 					  libpqres,
 					  PQerrorMessage(conn));
 		PQfinish(conn);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index fa58f86..54ef13f 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6264,6 +6264,15 @@ static struct config_int ConfigureNamesInt[] =
     },
 
     {
+            {"hawq_segment_history_keep_period", PGC_POSTMASTER, RESOURCES_MGM,
+                    gettext_noop("period of storing rows in segment_configuration_history"),
+                    NULL
+            },
+            &segment_history_keep_period,
+            365, 1, INT_MAX, NULL, NULL
+    },
+
+    {
             {"hawq_rm_master_domain_port", PGC_POSTMASTER, RESOURCES_MGM,
                     gettext_noop("resource manager master domain socket port number"),
                     NULL

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/64663eff/src/include/cdb/cdbvars.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index fbe6251..3129f88 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -1148,6 +1148,7 @@ extern int     seg_addr_port;
 extern char   *dfs_url;
 extern char   *master_directory;
 extern char   *seg_directory;
+extern int    segment_history_keep_period;
 
 /* HAWQ 2.0 resource manager GUCs */
 extern int	   rm_master_domain_port;


[47/50] incubator-hawq git commit: Revert "Revert "HAWQ-532. Optimise vseg number for copy to statement.""

Posted by sh...@apache.org.
Revert "Revert "HAWQ-532. Optimise vseg number for copy to statement.""

This reverts commit 86d9b0c03de349398ddb0337f7ac5dac3b58d2b9.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/695606ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/695606ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/695606ff

Branch: refs/heads/HAWQ-459
Commit: 695606ff442afcf4e9317451e02e3a6a31f900ff
Parents: 802d537
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 14:24:41 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Mon Mar 21 16:29:41 2016 +0800

----------------------------------------------------------------------
 src/backend/commands/copy.c | 97 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 89 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/695606ff/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index bcd5384..e37fb83 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,6 +91,11 @@
 #include "postmaster/autovacuum.h"
 #include "cdb/dispatcher.h"
 
+/*
+ * in dbsize.c
+ */
+extern int64 calculate_relation_size(Relation rel);
+
 /* DestReceiver for COPY (SELECT) TO */
 typedef struct
 {
@@ -137,6 +142,7 @@ static void copy_in_error_callback(void *arg);
 static void CopyInitPartitioningState(EState *estate);
 static void CopyInitDataParser(CopyState cstate);
 static bool CopyCheckIsLastLine(CopyState cstate);
+static int calculate_virtual_segment_number(List* candidateRelations);
 
 /* ==========================================================================
  * The follwing macros aid in major refactoring of data processing code (in
@@ -1556,7 +1562,16 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 
 			target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
 			Assert(target_policy);
-			target_segment_num = target_policy->bucketnum;
+			/*
+			 * For hash table we use table bucket number to request vsegs
+			 * For random table, we use a fixed GUC value to request vsegs.
+			 */
+			if(target_policy->nattrs > 0){
+				target_segment_num = target_policy->bucketnum;
+			}
+			else{
+				target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
+			}
 			pfree(target_policy);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1749,6 +1764,63 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 }
 
 /*
+ * calculate virtual segment number for copy statement.
+ * if there is hash distributed relations exist, use the max bucket number.
+ * if all relation are random, use the data size to determine vseg number.
+ */
+static int calculate_virtual_segment_number(List* candidateOids) {
+	ListCell* le1;
+	int vsegNumber = 1;
+	int64 totalDataSize = 0;
+	bool isHashRelationExist = false;
+	int maxHashBucketNumber = 0;
+
+	foreach (le1, candidateOids)
+	{
+		Oid				candidateOid	  = InvalidOid;
+		candidateOid = lfirst_oid(le1);
+
+		//Relation rel = (Relation)lfirst(le1);
+		Relation rel = relation_open(candidateOid, AccessShareLock);
+		if (candidateOid > 0 ) {
+			GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
+					candidateOid);
+			if(targetPolicy == NULL){
+				return GetAnalyzeVSegNumLimit();
+			}
+			if (targetPolicy->nattrs > 0) {
+				isHashRelationExist = true;
+				if(maxHashBucketNumber < targetPolicy->bucketnum){
+					maxHashBucketNumber = targetPolicy->bucketnum;
+				}
+			}
+			/*
+			 * if no hash relation, we calculate the data size of all the relations.
+			 */
+			if (!isHashRelationExist) {
+				totalDataSize += calculate_relation_size(rel);
+			}
+		}
+		relation_close(rel, AccessShareLock);
+	}
+
+	if (isHashRelationExist) {
+		vsegNumber = maxHashBucketNumber;
+	} else {
+		/*we allocate one virtual segment for each 128M data */
+		totalDataSize >>= 27;
+		vsegNumber = totalDataSize + 1;
+	}
+	Assert(vsegNumber > 0);
+	/*vsegNumber should be less than GetUtilPartitionNum*/
+	if(vsegNumber > GetQueryVsegNum()){
+		vsegNumber = GetQueryVsegNum();
+	}
+
+	return vsegNumber;
+}
+
+/*
  * This intermediate routine exists mainly to localize the effects of setjmp
  * so we don't need to plaster a lot of variables with "volatile".
  */
@@ -1809,13 +1881,22 @@ DoCopyTo(CopyState cstate)
 		 */
 		if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
 		{
-			GpPolicy *target_policy = NULL;
 			int target_segment_num = 0;
+			/*
+			 * copy hash table use table bucket number
+			 * copy random table use table size.
+			 */
+			PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
+													0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
+			List		*lFullRelOids = NIL;
+			if(pn){
+				lFullRelOids = all_leaf_partition_relids(pn);
+				lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
+			}
+			lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
 
-			target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
-			Assert(target_policy);
-			target_segment_num = target_policy->bucketnum;
-			pfree(target_policy);
+			target_segment_num = calculate_virtual_segment_number(lFullRelOids);
+			elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
 			CopyToDispatch(cstate);
@@ -4200,7 +4281,7 @@ CopyFrom(CopyState cstate)
 					if (cstate->oids && file_has_oids)
 						MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
 				}
-				else if ((relstorage == RELSTORAGE_PARQUET))
+				else if (relstorage == RELSTORAGE_PARQUET)
 				{
                     tuple = NULL;
 				}
@@ -4231,7 +4312,7 @@ CopyFrom(CopyState cstate)
 				{
 					HeapTuple	newtuple;
 
-					if((relstorage == RELSTORAGE_PARQUET))
+					if(relstorage == RELSTORAGE_PARQUET)
 					{
 						Assert(!tuple);
 						elog(ERROR, "triggers are not supported on tables that use column-oriented storage");


[04/50] incubator-hawq git commit: Revert "Support --enable-orca [#105338218]"

Posted by sh...@apache.org.
Revert "Support --enable-orca [#105338218]"

This reverts commit a48600a125eed2a3da00133d6f5f8dad22d4a6a4.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d5e7b927
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d5e7b927
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d5e7b927

Branch: refs/heads/HAWQ-459
Commit: d5e7b927020b80b812cd66f1416336d649f3bcc9
Parents: a48600a
Author: C.J. Jameson <cj...@pivotal.io>
Authored: Mon Mar 14 14:34:00 2016 -0700
Committer: C.J. Jameson <cj...@pivotal.io>
Committed: Mon Mar 14 14:34:00 2016 -0700

----------------------------------------------------------------------
 configure                          | 34 ++++++++++++++++++++++++++++++++
 configure.in                       | 35 +++++++++++++++++++++++++++++++++
 src/backend/Makefile               | 19 +++++++++---------
 src/backend/gpopt/Makefile         | 25 +++++++++++++----------
 src/backend/gpopt/gpopt.mk         | 24 ++++++++++------------
 src/bin/gpcheckhdfs/Makefile       |  2 +-
 src/bin/gpfilesystem/hdfs/Makefile |  2 +-
 7 files changed, 105 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5669406..5135536 100755
--- a/configure
+++ b/configure
@@ -731,6 +731,11 @@ with_perl
 with_tcl
 enable_thread_safety
 INCLUDES
+ORCA_DEPENDS_OPTIMIZER_VER
+ORCA_DEPENDS_LIBGPOS_VER
+ORCA_DEPENDS_XERCES_VER
+ORCA_DEPENDS_OBJDIR_DEFAULT
+ORCA_DEPENDS_DIR_INTER
 enable_orca
 enable_email
 enable_gphdfs
@@ -5517,6 +5522,35 @@ fi
 
 
 
+if test "$enable_orca" = yes ; then
+
+	if test -z "$ORCA_DEPENDS_DIR" ; then
+		as_fn_error $? "orca is enabled but ORCA_DEPENDS_DIR is not set" "$LINENO" 5]
+	fi
+
+	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
+		as_fn_error $? "orca is enabled but ORCA_OBJDIR_DEFAULT is not set" "$LINENO" 5]
+	fi
+
+	if test -z "$XERCES_VER" ; then
+		as_fn_error $? "orca is enabled but XERCES_VER is not set" "$LINENO" 5]
+	fi
+
+	if test -z "$LIBGPOS_VER" ; then
+		as_fn_error $? "orca is enabled but LIBGPOS_VER is not set" "$LINENO" 5]
+	fi
+
+	if test -z "$OPTIMIZER_VER" ; then
+		as_fn_error $? "orca is enabled but OPTIMIZER_VER is not set" "$LINENO" 5]
+	fi
+
+    ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
+    ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
+    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
+    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
+    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
+fi
+
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index 489187e..cda7bad 100644
--- a/configure.in
+++ b/configure.in
@@ -639,6 +639,41 @@ PGAC_ARG_BOOL(enable, orca, no, [  --enable-orca        enable Pivotal Query Opt
                          [Define to 1 to build with Pivotal Query Optimizer. (--enable-orca)])])
 AC_SUBST(enable_orca)
 
+if test "$enable_orca" = yes ; then
+
+	if test -z "$ORCA_DEPENDS_DIR" ; then
+		AC_MSG_ERROR([orca is enabled but ORCA_DEPENDS_DIR is not set])]
+	fi
+
+	if test -z "$ORCA_OBJDIR_DEFAULT" ; then
+		AC_MSG_ERROR([orca is enabled but ORCA_OBJDIR_DEFAULT is not set])]
+	fi
+	
+	if test -z "$XERCES_VER" ; then
+		AC_MSG_ERROR([orca is enabled but XERCES_VER is not set])]
+	fi
+	
+	if test -z "$LIBGPOS_VER" ; then
+		AC_MSG_ERROR([orca is enabled but LIBGPOS_VER is not set])]
+	fi
+	
+	if test -z "$OPTIMIZER_VER" ; then
+		AC_MSG_ERROR([orca is enabled but OPTIMIZER_VER is not set])]
+	fi
+
+    ORCA_DEPENDS_DIR_INTER="$ORCA_DEPENDS_DIR"
+    ORCA_DEPENDS_OBJDIR_DEFAULT="$ORCA_OBJDIR_DEFAULT"
+    ORCA_DEPENDS_XERCES_VER="$XERCES_VER"
+    ORCA_DEPENDS_LIBGPOS_VER="$LIBGPOS_VER"
+    ORCA_DEPENDS_OPTIMIZER_VER="$OPTIMIZER_VER"
+fi
+
+AC_SUBST(ORCA_DEPENDS_DIR_INTER)
+AC_SUBST(ORCA_DEPENDS_OBJDIR_DEFAULT)
+AC_SUBST(ORCA_DEPENDS_XERCES_VER)
+AC_SUBST(ORCA_DEPENDS_LIBGPOS_VER)
+AC_SUBST(ORCA_DEPENDS_OPTIMIZER_VER)
+
 #
 # Include directories
 #

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index f462510..8460b76 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -52,15 +52,15 @@ LIBS := $(filter-out -lpgport, $(LIBS)) $(LDAP_LIBS_BE) $(CONNECTEMCLIB)
 
 # The backend doesn't need everything that's in LIBS, however
 LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
-LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
+LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl -lxml2 $(LIBS)
 
 # adding orca libraries
 ifeq ($(enable_orca),yes)
-LIBS := $(LIBS) -lgpos
-LIBS := $(LIBS) -lxerces-c
-LIBS := $(LIBS) -lnaucrates
-LIBS := $(LIBS) -lgpdbcost
-LIBS := $(LIBS) -lgpopt
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpos/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libnaucrates/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost
+LIBS := $(LIBS) -L$(ORCA_DEPENDS_DIR_INTER)/libgpopt/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt
 LIBS := $(LIBS) -L$(top_builddir)/src/backend/gpopt -ldxltranslators
 endif
 
@@ -213,6 +213,9 @@ ifeq ($(MAKE_DLL), true)
 	$(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a'
 endif
 endif
+ifeq ($(enable_orca),yes)
+	$(MAKE) -C gpopt install
+endif
 	$(MAKE) -C catalog install-data
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_hba.conf.sample '$(DESTDIR)$(datadir)/pg_hba.conf.sample'
 	$(INSTALL_DATA) $(srcdir)/libpq/pg_ident.conf.sample '$(DESTDIR)$(datadir)/pg_ident.conf.sample'
@@ -225,10 +228,6 @@ endif
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/slaves ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/template-hawq-site.xml ${sysconfdir}
 	${INSTALL_DATA} $(srcdir)/utils/misc/etc/gpcheck.cnf ${sysconfdir}
-ifeq ($(enable_orca), yes)
-	$(MAKE) -C gpopt $@ INSTLOC=$(DESTDIR)$(libdir)
-endif
-
 
 install-bin: postgres $(POSTGRES_IMP) installdirs
 	$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/gpopt/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/Makefile b/src/backend/gpopt/Makefile
index a7ffa87..b587ffd 100644
--- a/src/backend/gpopt/Makefile
+++ b/src/backend/gpopt/Makefile
@@ -39,24 +39,29 @@ else
 	GPOPT_UDF_FLAGS := -shared
 endif
 
+LIBGPOS_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpos
+LIBNAUCRATES_DIR = $(ORCA_DEPENDS_DIR_INTER)/libnaucrates
+LIBGPDBCOST_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost
+LIBGPOPT_DIR = $(ORCA_DEPENDS_DIR_INTER)/libgpopt
+
 include $(top_srcdir)/src/backend/common.mk
 
 libdxltranslators.$(LDSFX): $(SUBDIROBJS) $(OBJS)
 	$(CXX) $(CXXFLAGS) $(LDLIBFLAGS) \
-	  -lgpos \
-	  -lxerces-c \
-	  -lnaucrates \
-	  -lgpdbcost \
-	  -lgpopt -lpthread -o $@ \
+	  -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
+	  -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
+	  -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
+	  -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
+	  -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt -lpthread -o $@ \
 	  $(SUBDIROBJS) $(OBJS)
 
 libgpoptudf.$(LDSFX): $(SUBDIROBJS)
 	$(CXX) $(CXXFLAGS) $(GPOPT_UDF_FLAGS) \
-	  -lxerces-c \
-	  -lgpos \
-	  -lnaucrates \
-	  -lgpdbcost \
-	  -lgpopt \
+	  -L$(ORCA_DEPENDS_DIR_INTER)/lib -lxerces-c \
+	  -L$(LIBGPOS_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpos \
+	  -L$(LIBNAUCRATES_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lnaucrates \
+	  -L$(LIBGPDBCOST_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpdbcost \
+	  -L$(LIBGPOPT_DIR)/$(ORCA_DEPENDS_OBJDIR_DEFAULT) -lgpopt \
 	  -o $@ ./utils/funcs.o
 
 install:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/backend/gpopt/gpopt.mk
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/gpopt.mk b/src/backend/gpopt/gpopt.mk
index 0099667..6c77d2f 100644
--- a/src/backend/gpopt/gpopt.mk
+++ b/src/backend/gpopt/gpopt.mk
@@ -25,8 +25,6 @@
 
 UNAME = $(shell uname)
 UNAME_P = $(shell uname -p)
-UNAME_M = $(shell uname -m)
-
 ARCH_OS = GPOS_$(UNAME)
 ARCH_CPU = GPOS_$(UNAME_P)
 
@@ -43,9 +41,8 @@ ifeq "$(BLD_TYPE)" "opt"
 	GPOPT_flags = -O3 -fno-omit-frame-pointer -g3
 endif
 
-ifeq (x86_64, $(UNAME_M))
-	ARCH_BIT = GPOS_64BIT
-else
+ARCH_BIT = GPOS_64BIT
+ifeq (Darwin, $(UNAME))
 	ARCH_BIT = GPOS_32BIT
 endif
 
@@ -55,15 +52,14 @@ else
 	ARCH_FLAGS = -m64
 endif
 
-GREP_SED_VAR = $(top_builddir)/src/backend/gpopt/ivy.xml | sed -e 's|\(.*\)rev="\(.*\)"[ ]*conf\(.*\)|\2|'
-
-XERCES_VER  = $(shell grep "\"xerces-c\""   $(GREP_SED_VAR))
-LIBGPOS_VER = $(shell grep "\"libgpos\""    $(GREP_SED_VAR))
-OPTIMIZER_VER = $(shell grep "\"optimizer\""  $(GREP_SED_VAR))
-
 BLD_FLAGS = $(ARCH_FLAGS) -D$(ARCH_BIT) -D$(ARCH_CPU) -D$(ARCH_OS) $(GPOPT_flags)
 override CPPFLAGS := -fPIC $(CPPFLAGS)
 override CPPFLAGS := $(BLD_FLAGS)  $(CPPFLAGS)
-override CPPFLAGS := -DGPOS_VERSION=\"$(LIBGPOS_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DGPOPT_VERSION=\"$(OPTIMIZER_VER)\" $(CPPFLAGS)
-override CPPFLAGS := -DXERCES_VERSION=\"$(XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOS_VERSION=\"$(ORCA_DEPENDS_LIBGPOS_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DGPOPT_VERSION=\"$(ORCA_DEPENDS_OPTIMIZER_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -DXERCES_VERSION=\"$(ORCA_DEPENDS_XERCES_VER)\" $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpos/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpopt/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libnaucrates/include $(CPPFLAGS)
+override CPPFLAGS := -I $(ORCA_DEPENDS_DIR_INTER)/libgpdbcost/include $(CPPFLAGS)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/bin/gpcheckhdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpcheckhdfs/Makefile b/src/bin/gpcheckhdfs/Makefile
index 0b88600..cd81902 100644
--- a/src/bin/gpcheckhdfs/Makefile
+++ b/src/bin/gpcheckhdfs/Makefile
@@ -20,7 +20,7 @@ subdir = src/bin/gpcheckhdfs
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-LIBS := -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
+LIBS := -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3 -lgsasl $(LIBS)
 
 OBJS=gpcheckhdfs.o 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d5e7b927/src/bin/gpfilesystem/hdfs/Makefile
----------------------------------------------------------------------
diff --git a/src/bin/gpfilesystem/hdfs/Makefile b/src/bin/gpfilesystem/hdfs/Makefile
index 2399789..b0fb359 100644
--- a/src/bin/gpfilesystem/hdfs/Makefile
+++ b/src/bin/gpfilesystem/hdfs/Makefile
@@ -24,7 +24,7 @@ OBJS       = gpfshdfs.o
 
 PG_CPPFLAGS = -I$(libpq_srcdir)
 PG_LIBS = $(libpq_pgport)
-SHLIB_LINK = -lprotobuf -lboost_thread-mt -lboost_system -lboost_date_time -lstdc++ -lhdfs3
+SHLIB_LINK = -lprotobuf -lboost_thread -lboost_system -lboost_date_time -lstdc++ -lhdfs3
 
 ifdef USE_PGXS
 PGXS := $(shell pg_config --pgxs)


[16/50] incubator-hawq git commit: HAWQ-529. Allocate resource for udf in resource negotiator.

Posted by sh...@apache.org.
HAWQ-529. Allocate resource for udf in resource negotiator.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8e38e844
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8e38e844
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8e38e844

Branch: refs/heads/HAWQ-459
Commit: 8e38e844f0d2a1c45f7d5cd25a8f5b639c8b6fb7
Parents: 31eb645
Author: hubertzhang <hz...@pivotal.io>
Authored: Wed Mar 16 10:51:07 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Wed Mar 16 10:51:07 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c    | 194 ++++++++++++++++++++----------
 src/backend/executor/spi.c           |  72 +++++++++--
 src/backend/optimizer/plan/planner.c |  57 ++++++++-
 src/backend/optimizer/util/clauses.c |  11 +-
 src/backend/optimizer/util/walkers.c |   1 +
 src/include/cdb/cdbdatalocality.h    |  19 ++-
 src/include/executor/spi.h           |   7 ++
 src/include/optimizer/planner.h      |   4 +
 8 files changed, 289 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index c1f829b..d15d6b0 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,6 +33,9 @@
 #include "access/filesplit.h"
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
+#include "catalog/catquery.h"
+#include "catalog/pg_inherits.h"
+#include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
 #include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
@@ -41,6 +44,7 @@
 #include "utils/tqual.h"
 #include "utils/memutils.h"
 #include "executor/execdesc.h"
+#include "executor/spi.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "optimizer/walkers.h"
@@ -54,11 +58,7 @@
 #include "catalog/pg_proc.h"
 #include "postgres.h"
 #include "resourcemanager/utils/hashtable.h"
-#include "catalog/pg_inherits.h"
-//#include "utils/misc/guc.c"
 
-#define PRONAME 1
-#define PROISAGG 5
 /* We need to build a mapping from host name to host index */
 
 extern bool		optimizer; /* Enable the optimizer */
@@ -372,7 +372,7 @@ static Block_Host_Index * update_data_dist_stat(
 static HostDataVolumeInfo *search_host_in_stat_context(
 		split_to_segment_mapping_context *context, char *hostname);
 
-static bool IsAggFunction(char* funcName);
+static bool IsBuildInFunction(Oid funcOid);
 
 static bool allocate_hash_relation(Relation_Data* rel_data,
 		Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -630,39 +630,36 @@ static void collect_range_tables(Query *query, List* full_range_table,
 /*
  *
  */
-static bool IsAggFunction(char* funcName) {
-	if (funcName == NULL) {
-		return false;
-	}
-	Relation pg_proc_rel;
-	TupleDesc pg_proc_dsc;
-	HeapTuple tuple;
-	SysScanDesc pg_proc_scan;
-
-	pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
-	pg_proc_dsc = RelationGetDescr(pg_proc_rel);
-	ScanKeyData skey;
+static bool IsBuildInFunction(Oid foid) {
 
-	ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
-	F_NAMEEQ, CStringGetDatum(funcName));
-
-	pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
-			ActiveSnapshot, 1, &skey);
-	while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+	cqContext  *pcqCtx;
+	HeapTuple readtup = NULL;
+	HeapTuple	procedureTuple;
+	Form_pg_proc procedureStruct;
 
-		bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
-		systable_endscan(pg_proc_scan);
-		heap_close(pg_proc_rel, AccessShareLock);
-		if (isAgg) {
-			return true;
-		} else {
-			return false;
-		}
+	/*
+	 * get the procedure tuple corresponding to the given function Oid
+	 */
+	pcqCtx = caql_beginscan(
+			NULL,
+			cql("SELECT * FROM pg_proc "
+				" WHERE oid = :1 ",
+				ObjectIdGetDatum(foid)));
+
+	procedureTuple = caql_getnext(pcqCtx);
+
+	if (!HeapTupleIsValid(procedureTuple))
+		elog(ERROR, "cache lookup failed for function %u", foid);
+	procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
+	caql_endscan(pcqCtx);
+	/* we treat proc namespace = 11 to build in function.*/
+	if (procedureStruct->pronamespace == 11) {
+		return true;
+	} else {
+		return false;
 	}
-	systable_endscan(pg_proc_scan);
-	heap_close(pg_proc_rel, AccessShareLock);
-	return true;
 }
+
 /*
  *
  */
@@ -676,25 +673,6 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
 	foreach(old_lc, *range_tables)
 	{
 		RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
-		if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
-			*isTableFunctionExists = true;
-		}
-		if (entry->rtekind == RTE_SUBQUERY) {
-			Query* subQuery = entry->subquery;
-			ListCell *lc;
-			foreach(lc, subQuery->targetList)
-			{
-				TargetEntry *te = (TargetEntry *) lfirst(lc);
-				bool isAggFunc = IsAggFunction(te->resname);
-				// if target list of subquery contains non aggregate function,
-				// then we consider the query contains and use default_segment_num guc
-				// as the number of virtual segment
-				if (!isAggFunc) {
-					*isTableFunctionExists = true;
-				}
-			}
-
-		}
 		if (entry->rtekind != RTE_RELATION) {
 			continue;
 		}
@@ -3958,11 +3936,54 @@ static void cleanup_allocation_algorithm(
 }
 
 /*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,
+		udf_collector_context *context) {
+	if (node == NULL) {
+		return false;
+	}
+
+	if (IsA(node, Query)) {
+		return query_tree_walker((Query *) node, udf_collector_walker,
+				(void *) context,
+				QTW_EXAMINE_RTES);
+	}
+
+	/*For Aggref, we don't consider it as udf.*/
+
+	if(IsA(node,FuncExpr)){
+		if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
+			context->udf_exist = true;
+		}
+		return false;
+	}
+
+	return expression_tree_walker(node, udf_collector_walker,
+			(void *) context);
+
+	return false;
+}
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context) {
+
+	query_tree_walker(query, udf_collector_walker, (void *) context,
+	QTW_EXAMINE_RTES);
+
+	return;
+}
+
+
+/*
  * calculate_planner_segment_num
+ * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
  */
 SplitAllocResult *
 calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
+		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
 	SplitAllocResult *result = NULL;
 	QueryResource *resource = NULL;
 	QueryResourceParameters *resource_parameters = NULL;
@@ -3971,14 +3992,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	List *alloc_result = NIL;
 	split_to_segment_mapping_context context;
 
-	int planner_segments = -1; /*virtual segments number for explain statement */
+	int planner_segments = 0; /*virtual segments number for explain statement */
 
 	result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
 	result->resource = NULL;
 	result->resource_parameters = NULL;
 	result->alloc_results = NIL;
 	result->relsType = NIL;
-	result->planner_segments = -1;
+	result->planner_segments = 0;
 	result->datalocalityInfo = makeStringInfo();
 
 	/* fake data locality */
@@ -3995,7 +4016,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		result->resource_parameters = NULL;
 		result->alloc_results = NIL;
 		result->relsType = NIL;
-		result->planner_segments = -1;
+		result->planner_segments = 0;
 		return result;
 	}
 
@@ -4020,6 +4041,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		 * 5 data size of random "from" relation
 		 */
 
+		udf_collector_context udf_context;
+		udf_context.udf_exist = false;
+
+		find_udf(query, &udf_context);
+		isTableFunctionExists = udf_context.udf_exist;
 		/*convert range table list to oid list and check whether table function exists
 		 *we keep a full range table list and a range table list without result relation separately
 		 */
@@ -4046,7 +4072,15 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 		/*use inherit resource*/
 		if (resourceLife == QRL_INHERIT) {
-			resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+
+			if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
+			{
+				resource = NULL;
+			}
+			else
+			{
+				resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+			}
 
 			saveQueryResourceParameters(
 							resource_parameters,  /* resource_parameters */
@@ -4197,6 +4231,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				maxTargetSegmentNumber = enforce_virtual_segment_number;
 				minTargetSegmentNumber = enforce_virtual_segment_number;
 			}
+			/* in PBE mode, the execute should use the same vseg number. */
+			if(fixedVsegNum > 0 ){
+				maxTargetSegmentNumber = fixedVsegNum;
+				minTargetSegmentNumber = fixedVsegNum;
+			}
 			uint64_t before_rm_allocate_resource = gettime_microsec();
 
 			/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4204,9 +4243,40 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			mincost <<= 20;
 			int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
 			if (QRL_NONE != resourceLife) {
-				resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
-						maxTargetSegmentNumber, minTargetSegmentNumber,
-						context.host_context.hostnameVolInfos, context.host_context.size);
+
+				if (SPI_IsInPrepare())
+				{
+					resource = NULL;
+					/*
+					 * prepare need to get resource quota from RM
+					 * and pass quota(planner_segments) to Orca or Planner to generate plan
+					 * the following executes(in PBE) should reallocate the same number
+					 * of resources.
+					 */
+					uint32 seg_num;
+					uint32 seg_num_min;
+					uint32 seg_memory_mb;
+					double seg_core;
+
+					GetResourceQuota(maxTargetSegmentNumber,
+					                 minTargetSegmentNumber,
+					                 &seg_num,
+					                 &seg_num_min,
+					                 &seg_memory_mb,
+					                 &seg_core);
+
+					planner_segments = seg_num;
+					minTargetSegmentNumber = planner_segments;
+					maxTargetSegmentNumber = planner_segments;
+				}
+				else
+				{
+					resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+					                            maxTargetSegmentNumber,
+					                            minTargetSegmentNumber,
+					                            context.host_context.hostnameVolInfos,
+					                            context.host_context.size);
+				}
 
 				saveQueryResourceParameters(
 								resource_parameters,                   /* resource_parameters */
@@ -4236,7 +4306,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 			if (resource == NULL) {
 				result->resource = NULL;
-				result->resource_parameters = NULL;
+				result->resource_parameters = resource_parameters;
 				result->alloc_results = NIL;
 				result->relsType = NIL;
 				result->planner_segments = planner_segments;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 07812aa..8e7645c 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,6 +46,7 @@
 #include "cdb/memquota.h"
 #include "executor/nodeFunctionscan.h"
 #include "nodes/stack.h"
+#include "cdb/cdbdatalocality.h"
 
 extern char *savedSeqServerHost;
 extern int savedSeqServerPort;
@@ -74,6 +75,7 @@ static int	_SPI_curid = -1;
 
 static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
 static char *_QD_currently_prepared_stmt = NULL;
+static int SPI_prepare_depth = 0;
 
 static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
 
@@ -106,6 +108,32 @@ static bool _SPI_checktuples(void);
 
 /* =================== interface functions =================== */
 
+bool SPI_IsInPrepare(void)
+{
+	if (SPI_prepare_depth > 0)
+	{
+		return true;
+	}
+	else if (SPI_prepare_depth < 0)
+	{
+		elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
+		            SPI_prepare_depth);
+	}
+
+	return false;
+}
+
+void SPI_IncreasePrepareDepth(void)
+{
+	SPI_prepare_depth++;
+}
+
+void SPI_DecreasePrepareDepth(void)
+{
+	SPI_prepare_depth--;
+}
+
+
 int
 SPI_connect(void)
 {
@@ -566,6 +594,8 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 	_SPI_plan	plan;
 	_SPI_plan  *result;
 
+	SPI_IncreasePrepareDepth();
+
 	if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
 	{
 		SPI_result = SPI_ERROR_ARGUMENT;
@@ -591,9 +621,13 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 
 		/* copy plan to procedure context */
 		result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
+
+		SPI_DecreasePrepareDepth();
 	}
 	PG_CATCH();
 	{
+		SPI_DecreasePrepareDepth();
+
 		_SPI_end_call(true);
 		PG_RE_THROW();
 	}
@@ -1819,13 +1853,37 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 					     (stmt->resource == NULL) &&
 					     (stmt->resource_parameters != NULL) )
 					{
-						stmt->resource = AllocateResource(stmt->resource_parameters->life,
-						                        stmt->resource_parameters->slice_size,
-						                        stmt->resource_parameters->iobytes,
-						                        stmt->resource_parameters->max_target_segment_num,
-						                        stmt->resource_parameters->min_target_segment_num,
-						                        stmt->resource_parameters->vol_info,
-						                        stmt->resource_parameters->vol_info_size);
+						SplitAllocResult *allocResult = NULL;
+
+						/* If this is a parallel plan. */
+						if (stmt->planTree->dispatch == DISPATCH_PARALLEL)
+						{
+							/*
+							 * Now, we want to allocate resource.
+							 */
+							allocResult = calculate_planner_segment_num(queryTree,
+							                                            stmt->resource_parameters->life,
+							                                            stmt->rtable,
+							                                            stmt->intoPolicy,
+							                                            stmt->nMotionNodes + stmt->nInitPlans + 1,
+							                                            stmt->resource_parameters->min_target_segment_num);
+
+							Assert(allocResult);
+
+							if(stmt->resource !=NULL)
+							{
+								pfree(stmt->resource);
+							}
+							stmt->resource = allocResult->resource;
+							if(stmt->scantable_splits !=NULL)
+							{
+								list_free_deep(stmt->scantable_splits);
+							}
+							stmt->scantable_splits = allocResult->alloc_results;
+							stmt->planner_segments = allocResult->planner_segments;
+							stmt->datalocalityInfo = allocResult->datalocalityInfo;
+							pfree(allocResult);
+						}
 					}
 
 					originalStmt->resource = NULL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index be057e0..c6be59a 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,6 +91,8 @@ planner_hook_type planner_hook = NULL;
 
 ParamListInfo PlannerBoundParamList = NULL;		/* current boundParams */
 
+static int PlanningDepth = 0;		/* Planning depth */
+
 /* Expression kind codes for preprocess_expression */
 #define EXPRKIND_QUAL			0
 #define EXPRKIND_TARGET			1
@@ -165,6 +167,32 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
 static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
 bool is_dummy_plan(Plan *plan);
 
+
+bool is_in_planning_phase(void)
+{
+	if (PlanningDepth > 0)
+	{
+		return true;
+	}
+	else if (PlanningDepth < 0)
+	{
+		elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
+	}
+
+	return false;
+}
+
+void increase_planning_depth(void)
+{
+	PlanningDepth++;
+}
+
+void decrease_planning_depth(void)
+{
+	PlanningDepth--;
+}
+
+
 #ifdef USE_ORCA
 /**
  * Logging of optimization outcome
@@ -285,7 +313,7 @@ planner(Query *parse, int cursorOptions,
 	PlannedStmt *result = NULL;
 	instr_time	starttime, endtime;
 	ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
-	SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
+	SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
 	ppResult->saResult = initResult;
 	ppResult->stmt = NULL;
 	static int plannerLevel = 0;
@@ -300,6 +328,8 @@ planner(Query *parse, int cursorOptions,
 	 * resource to run this query. After gaining the resource, we can perform the
 	 * actual optimization.
 	 */
+	increase_planning_depth();
+
 	plannerLevel++;
 	if (!resourceNegotiateDone)
 	{
@@ -309,6 +339,8 @@ planner(Query *parse, int cursorOptions,
       {
         resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
 
+		decrease_planning_depth();
+
 		if(ppResult->stmt && ppResult->stmt->planTree)
 		{
 			isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -318,6 +350,8 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  PG_CATCH();
 	  {
+		decrease_planning_depth();
+
 		if ((ppResult != NULL))
 		{
 		  pfree(ppResult);
@@ -430,7 +464,7 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  else
 	  {
-	    gp_segments_for_planner = -1;
+	    gp_segments_for_planner = 0;
 	  }
 	  SetActiveQueryResource(savedQueryResource);
 	  if ((ppResult != NULL))
@@ -450,7 +484,7 @@ planner(Query *parse, int cursorOptions,
 		}
 		else
 		{
-			gp_segments_for_planner = -1;
+			gp_segments_for_planner = 0;
 		}
 		SetActiveQueryResource(savedQueryResource);
 
@@ -470,6 +504,7 @@ planner(Query *parse, int cursorOptions,
 	return result;
 }
 
+
 /*
  * The new framework for HAWQ 2.0 query optimizer
  */
@@ -480,6 +515,8 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
   PlannedStmt *plannedstmt = NULL;
   do
   {
+  		udf_collector_context udf_context;
+  		udf_context.udf_exist = false;
     SplitAllocResult *allocResult = NULL;
     Query *my_parse = copyObject(parse);
     ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -495,12 +532,24 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
        */
       allocResult = calculate_planner_segment_num(my_parse, resourceLife,
                                           plannedstmt->rtable, plannedstmt->intoPolicy,
-                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
+                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
+                                          -1);
 
       Assert(allocResult);
 
       (*result)->saResult = *allocResult;
       pfree(allocResult);
+    }else{
+    		find_udf(my_parse, &udf_context);
+    		if(udf_context.udf_exist){
+    			if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
+    				int64 mincost = min_cost_for_each_query;
+    				mincost <<= 20;
+    				int avgSliceNum = 3;
+    				(*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
+    						GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
+    			}
+    		}
     }
   } while (0);
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a80737f..a3d67f2 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,8 +2831,15 @@ simplify_function(Oid funcid, Oid result_type, List *args,
 	if (!HeapTupleIsValid(func_tuple))
 		elog(ERROR, "cache lookup failed for function %u", funcid);
 
-	newexpr = evaluate_function(funcid, result_type, args,
-								func_tuple, context);
+	if (is_in_planning_phase())
+	{
+		newexpr = NULL;
+	}
+	else
+	{
+		newexpr = evaluate_function(funcid, result_type, args,
+		                            func_tuple, context);
+	}
 
 	if (large_const(newexpr, context->max_size))
 	{

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index 6f1ee51..a8222fb 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,6 +162,7 @@ expression_tree_walker(Node *node,
 		case T_PartBoundExpr:
 		case T_PartBoundInclusionExpr:
 		case T_PartBoundOpenExpr:
+		case T_RangeTblEntry	:
 			/* primitive node types with no expression subnodes */
 			break;
 		case T_Aggref:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index 95f2bd5..67375d7 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -49,6 +49,13 @@ typedef struct SplitAllocResult
 } SplitAllocResult;
 
 /*
+ * structure containing all relation range table entries.
+ */
+typedef struct udf_collector_context {
+	bool udf_exist;
+} udf_collector_context;
+
+/*
  * structure containing rel and type when execution
  */
 typedef struct CurrentRelType {
@@ -86,7 +93,17 @@ void saveQueryResourceParameters(
  * we calculate the appropriate planner segment_num.
  */
 SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-                                                List *rtable, GpPolicy *intoPolicy, int sliceNum);
+                                                List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
+
+/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,	udf_collector_context *context);
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context);
 
 FILE *fp;
 FILE *fpaoseg;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 867daf5..7b4ba1b 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,4 +155,11 @@ extern uint64 SPI_GetMemoryReservation(void);
 extern void SPI_ReserveMemory(uint64 mem_reserved);
 extern bool SPI_IsMemoryReserved(void);
 
+/**
+ * Query resource related routines.
+ */
+extern bool SPI_IsInPrepare(void);
+extern void SPI_IncreasePrepareCounter(void);
+extern void SPI_DecreasePrepareCounter(void);
+
 #endif   /* SPI_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8e38e844/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index 48de348..cc2eb7d 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,4 +50,8 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
 								   double dNumGroups, 
 								   AggClauseCounts *agg_counts);
 
+extern bool is_in_planning_phase(void);
+extern void increase_planning_depth(void);
+extern void decrease_planning_depth(void);
+
 #endif   /* PLANNER_H */


[32/50] incubator-hawq git commit: HAWQ-554. Create web external table shouldnot omit ON clause.

Posted by sh...@apache.org.
HAWQ-554. Create web external table shouldnot omit ON clause.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/17af44cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/17af44cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/17af44cb

Branch: refs/heads/HAWQ-459
Commit: 17af44cb139ddbe6b0e903d041d1177b73d2e556
Parents: 2513ad1
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:33:46 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800

----------------------------------------------------------------------
 src/backend/parser/gram.y                        | 9 ++++++---
 src/test/regress/output/exttab1.source           | 6 ++++--
 src/test/regress/output/exttab1_optimizer.source | 6 ++++--
 3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/backend/parser/gram.y
----------------------------------------------------------------------
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 4d6f76f..dc0e13b 100755
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -4610,9 +4610,12 @@ CreateExternalStmt:	CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n
 								
 								/* if no ON clause specified, default to "ON ALL" */
 								if(extdesc->on_clause == NIL)
-								{									
-									extdesc->on_clause = lappend(extdesc->on_clause, 
-										   				   		 makeDefElem("all", (Node *)makeInteger(TRUE)));
+								{
+									ereport(ERROR,
+										(errcode(ERRCODE_SYNTAX_ERROR),
+										errmsg("ON clause must be specified in external table"),
+										errhint("Use ON number instread"),
+										errOmitLocation(true)));
 								}
 							}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/exttab1.source b/src/test/regress/output/exttab1.source
index bd9dec8..ec8ed3b 100755
--- a/src/test/regress/output/exttab1.source
+++ b/src/test/regress/output/exttab1.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
 -- test for exec child process stderr showing in error message
 --
 create external web table ext_stderr1(a text) execute 'nosuchcommand' format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 --
 -- bad csv (quote must be a single char)
 --

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/17af44cb/src/test/regress/output/exttab1_optimizer.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/exttab1_optimizer.source b/src/test/regress/output/exttab1_optimizer.source
index 9519769..3f03143 100755
--- a/src/test/regress/output/exttab1_optimizer.source
+++ b/src/test/regress/output/exttab1_optimizer.source
@@ -557,9 +557,11 @@ drop external table ext_mpp12839;
 -- test for exec child process stderr showing in error message
 --
 create external web table ext_stderr1(a text) execute 'nosuchcommand' format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 create external web table ext_stderr2(a text) execute 'cat nosuchfile.txt' format 'text';
-ERROR:  the ON ALL syntax for external tables is deprecated
+ERROR:  ON clause must be specified in external table
+HINT:  Use ON number instread
 --
 -- bad csv (quote must be a single char)
 --


[42/50] incubator-hawq git commit: HAWQ-561. Wrongly check variance of number of virtual segments

Posted by sh...@apache.org.
HAWQ-561. Wrongly check variance of number of virtual segments


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1d9eb5e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1d9eb5e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1d9eb5e3

Branch: refs/heads/HAWQ-459
Commit: 1d9eb5e32a9c0a6a46e39f854b40442b5b0d12c8
Parents: 86d9b0c
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 19:18:39 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 19:18:39 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcepool.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1d9eb5e3/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index e5d083c..acc8dbe 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2556,7 +2556,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 		freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
 		Assert(minnvseg <= maxnvseg);
 
-		if ( maxnvseg - minnvseg > rm_tolerate_nseg_limit )
+		if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_limit )
 		{
 			elog(LOG, "Reject virtual segment allocation based on data "
 					  "locality information. After tentative allocation "
@@ -2565,7 +2565,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 					  "is %d, tolerated difference limit is %d.",
 					  maxnvseg,
 					  minnvseg,
-					  rm_tolerate_nseg_limit);
+					  rm_nvseg_variance_among_seg_limit);
 
 			/* Return the allocated resource. */
 			List 	 *vsegcntlist = NULL;


[12/50] incubator-hawq git commit: HAWQ-541. When YARN containers disappear from container report, the segment available resource maybe negative

Posted by sh...@apache.org.
HAWQ-541. When YARN containers disappear from container report, the segment available resource maybe negative


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/99a307b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/99a307b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/99a307b1

Branch: refs/heads/HAWQ-459
Commit: 99a307b185ce0d3821a736ab39855eb4ee08541f
Parents: 5152fb0
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Mar 15 14:30:05 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Mar 15 14:30:05 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcepool.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/99a307b1/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 6a36801..d3a01a4 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -2396,7 +2396,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 			int segcountact = containerset == NULL ?
 							  0 :
 							  containerset->Available.MemoryMB / memory;
-			if ( segcountact == 0 )
+			if ( segcountact <= 0 )
 			{
 				elog(RMLOG, "Segment %s does not have more resource to allocate. "
 							"This segment is skipped.",
@@ -3838,16 +3838,12 @@ void checkGRMContainerStatus(RB_GRMContainerStat ctnstats, int size)
 
 					Assert( segres->Allocated.MemoryMB >= 0 );
 					Assert( segres->Allocated.Core     >= 0 );
-					Assert( segres->Available.MemoryMB >= 0 );
-					Assert( segres->Available.Core     >= 0 );
 
 					minusResourceBundleData(&(ctnset->Allocated), ctn->MemoryMB, ctn->Core);
 					minusResourceBundleData(&(ctnset->Available), ctn->MemoryMB, ctn->Core);
 
 					Assert( ctnset->Allocated.MemoryMB >= 0 );
 					Assert( ctnset->Allocated.Core     >= 0 );
-					Assert( ctnset->Available.MemoryMB >= 0 );
-					Assert( ctnset->Available.Core     >= 0 );
 
 					reorderSegResourceAllocIndex(segres, PQUEMGR->RatioReverseIndex[ridx]);
 					reorderSegResourceAvailIndex(segres, PQUEMGR->RatioReverseIndex[ridx]);


[02/50] incubator-hawq git commit: HAWQ-523. Remove dead code in executormgr

Posted by sh...@apache.org.
HAWQ-523. Remove dead code in executormgr


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/29a9beb1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/29a9beb1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/29a9beb1

Branch: refs/heads/HAWQ-459
Commit: 29a9beb1df109815deebf3750c23e2386fa5377e
Parents: b3e2add
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 11:36:19 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Mon Mar 14 14:00:14 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/executormgr.c | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/29a9beb1/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 4d1351d..88dd32f 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -202,13 +202,9 @@ executormgr_bind_executor_task(struct DispatchData *data,
 							struct DispatchTask *task,
 							struct DispatchSlice *slice)
 {
-	executor->state = QES_UNINIT;
-	if (desc == NULL)
-	{
-		executor->health = QEH_ERROR;
-		return false;
-	}
+  Assert(desc != NULL);
 
+  executor->state = QES_UNINIT;
 	executor->desc = desc;
 	executor->state = QES_DISPATCHABLE;
 	executor->health = QEH_NA;


[34/50] incubator-hawq git commit: HAWQ-529. Allocate resource for udf in resource negotiator.

Posted by sh...@apache.org.
HAWQ-529. Allocate resource for udf in resource negotiator.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d997b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d997b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d997b57

Branch: refs/heads/HAWQ-459
Commit: 4d997b5714bb09ed9020a56acfce13b1d6fd637c
Parents: 2c033d9
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:14:42 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c    | 213 ++++++++++++++++++++----------
 src/backend/executor/spi.c           |  71 ++++++++--
 src/backend/optimizer/plan/planner.c |  59 ++++++++-
 src/backend/optimizer/util/clauses.c |  11 +-
 src/backend/optimizer/util/walkers.c |   1 +
 src/include/cdb/cdbdatalocality.h    |  19 ++-
 src/include/executor/spi.h           |   7 +
 src/include/optimizer/planner.h      |   4 +
 8 files changed, 294 insertions(+), 91 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index eec87b4..3f8fcb5 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -33,6 +33,9 @@
 #include "access/filesplit.h"
 #include "access/parquetsegfiles.h"
 #include "catalog/catalog.h"
+#include "catalog/catquery.h"
+#include "catalog/pg_inherits.h"
+#include "catalog/pg_proc.h"
 #include "cdb/cdbdatalocality.h"
 #include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
@@ -41,6 +44,7 @@
 #include "utils/tqual.h"
 #include "utils/memutils.h"
 #include "executor/execdesc.h"
+#include "executor/spi.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "optimizer/walkers.h"
@@ -54,11 +58,7 @@
 #include "catalog/pg_proc.h"
 #include "postgres.h"
 #include "resourcemanager/utils/hashtable.h"
-#include "catalog/pg_inherits.h"
-//#include "utils/misc/guc.c"
 
-#define PRONAME 1
-#define PROISAGG 5
 /* We need to build a mapping from host name to host index */
 
 extern bool		optimizer; /* Enable the optimizer */
@@ -376,7 +376,7 @@ static Block_Host_Index * update_data_dist_stat(
 static HostDataVolumeInfo *search_host_in_stat_context(
 		split_to_segment_mapping_context *context, char *hostname);
 
-static bool IsAggFunction(char* funcName);
+static bool IsBuildInFunction(Oid funcOid);
 
 static bool allocate_hash_relation(Relation_Data* rel_data,
 		Assignment_Log_Context *log_context, TargetSegmentIDMap* idMap,
@@ -638,39 +638,35 @@ static void collect_range_tables(Query *query, List* full_range_table,
 /*
  *
  */
-static bool IsAggFunction(char* funcName) {
-	if (funcName == NULL) {
-		return false;
-	}
-	Relation pg_proc_rel;
-	TupleDesc pg_proc_dsc;
-	HeapTuple tuple;
-	SysScanDesc pg_proc_scan;
-
-	pg_proc_rel = heap_open(ProcedureRelationId, AccessShareLock);
-	pg_proc_dsc = RelationGetDescr(pg_proc_rel);
-	ScanKeyData skey;
+static bool IsBuildInFunction(Oid foid) {
 
-	ScanKeyInit(&skey, PRONAME, BTEqualStrategyNumber,
-	F_NAMEEQ, CStringGetDatum(funcName));
-
-	pg_proc_scan = systable_beginscan(pg_proc_rel, InvalidOid, FALSE,
-			ActiveSnapshot, 1, &skey);
-	while (HeapTupleIsValid(tuple = systable_getnext(pg_proc_scan))) {
+	cqContext  *pcqCtx;
+	HeapTuple	procedureTuple;
+	Form_pg_proc procedureStruct;
 
-		bool isAgg = DatumGetBool(fastgetattr(tuple, PROISAGG, pg_proc_dsc, NULL));
-		systable_endscan(pg_proc_scan);
-		heap_close(pg_proc_rel, AccessShareLock);
-		if (isAgg) {
-			return true;
-		} else {
-			return false;
-		}
+	/*
+	 * get the procedure tuple corresponding to the given function Oid
+	 */
+	pcqCtx = caql_beginscan(
+			NULL,
+			cql("SELECT * FROM pg_proc "
+				" WHERE oid = :1 ",
+				ObjectIdGetDatum(foid)));
+
+	procedureTuple = caql_getnext(pcqCtx);
+
+	if (!HeapTupleIsValid(procedureTuple))
+		elog(ERROR, "cache lookup failed for function %u", foid);
+	procedureStruct = (Form_pg_proc) GETSTRUCT(procedureTuple);
+	caql_endscan(pcqCtx);
+	/* we treat proc namespace = 11 to build in function.*/
+	if (procedureStruct->pronamespace == 11) {
+		return true;
+	} else {
+		return false;
 	}
-	systable_endscan(pg_proc_scan);
-	heap_close(pg_proc_rel, AccessShareLock);
-	return true;
 }
+
 /*
  *
  */
@@ -684,25 +680,6 @@ static void convert_range_tables_to_oids_and_check_table_functions(List **range_
 	foreach(old_lc, *range_tables)
 	{
 		RangeTblEntry *entry = (RangeTblEntry *) lfirst(old_lc);
-		if (entry->rtekind == RTE_FUNCTION || entry->rtekind == RTE_TABLEFUNCTION) {
-			*isTableFunctionExists = true;
-		}
-		if (entry->rtekind == RTE_SUBQUERY) {
-			Query* subQuery = entry->subquery;
-			ListCell *lc;
-			foreach(lc, subQuery->targetList)
-			{
-				TargetEntry *te = (TargetEntry *) lfirst(lc);
-				bool isAggFunc = IsAggFunction(te->resname);
-				// if target list of subquery contains non aggregate function,
-				// then we consider the query contains and use default_segment_num guc
-				// as the number of virtual segment
-				if (!isAggFunc) {
-					*isTableFunctionExists = true;
-				}
-			}
-
-		}
 		if (entry->rtekind != RTE_RELATION) {
 			continue;
 		}
@@ -1833,7 +1810,7 @@ static int select_random_host_algorithm(Relation_Assignment_Context *context,
 	}
 	if (debug_fake_datalocality) {
 		fprintf(fp,
-				"cur_size_of_whole_query is:"INT64_FORMAT", avg_size_of_whole_query is: %.3f",
+				"cur_size_of_whole_query is:%.0f, avg_size_of_whole_query is: %.3f",
 				context->totalvols_with_penalty[minindex] + net_disk_ratio * splitsize,
 				context->avg_size_of_whole_query);
 	}
@@ -2393,7 +2370,7 @@ static Relation_File** change_file_order_based_on_continuity(
 		Relation_Data *rel_data, TargetSegmentIDMap* idMap, int host_num,
 		int* fileCount, Relation_Assignment_Context *assignment_context) {
 
-	Relation_File** file_vector;
+	Relation_File** file_vector = NULL;
 	int* isBlockContinue = (int *) palloc(sizeof(int) * host_num);
 	for (int i = 0; i < host_num; i++) {
 		isBlockContinue[i] = 0;
@@ -3429,45 +3406,45 @@ static void print_datalocality_overall_log_information(SplitAllocResult *result,
 			if(log_context->minSegmentNumofHost > 0 ){
 				fprintf(fpratio, "segmentnumber_perhost_max/min=%.2f\n", (double)(log_context->maxSegmentNumofHost / log_context->minSegmentNumofHost));
 			}else{
-				fprintf(fpratio, "segmentnumber_perhost_max/min="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segmentnumber_perhost_max/min=%lld\n", INT64_MAX);
 			}
 			if(log_context->avgSegmentNumofHost > 0 ){
 				fprintf(fpratio, "segmentnumber_perhost_max/avg=%.2f\n", (double)(log_context->maxSegmentNumofHost / log_context->avgSegmentNumofHost));
 			}else{
-				fprintf(fpratio, "segmentnumber_perhost_max/avg="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segmentnumber_perhost_max/avg=%lld\n", INT64_MAX);
 			}
 
 			if (log_context->minSizeSegmentOverall > 0){
 				fprintf(fpratio, "segments_size_max/min=%.5f\n", (double)log_context->maxSizeSegmentOverall / (double)log_context->minSizeSegmentOverall);
 			}else{
-				fprintf(fpratio, "segments_size_max/min="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segments_size_max/min=%lld\n", INT64_MAX);
 			}
 			if (log_context->avgSizeOverall > 0){
 				fprintf(fpratio, "segments_size_max/avg=%.5f\n", log_context->maxSizeSegmentOverall / log_context->avgSizeOverall);
 			}else{
-				fprintf(fpratio, "segments_size_max/avg="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segments_size_max/avg=%lld\n", INT64_MAX);
 			}
 
 			if (log_context->minSizeSegmentOverallPenalty > 0){
 				fprintf(fpratio, "segments_size_penalty_max/min=%.5f\n",(double)log_context->maxSizeSegmentOverallPenalty / (double)log_context->minSizeSegmentOverallPenalty);
 			}else{
-				fprintf(fpratio, "segments_size_penalty_max/min="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segments_size_penalty_max/min=%lld\n", INT64_MAX);
 			}
 			if (log_context->avgSizeOverallPenalty > 0){
 				fprintf(fpratio, "segments_size_penalty_max/avg=%.5f\n",log_context->maxSizeSegmentOverallPenalty / log_context->avgSizeOverallPenalty);
 			}else{
-				fprintf(fpratio, "segments_size_penalty_max/avg="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "segments_size_penalty_max/avg=%lld\n", INT64_MAX);
 			}
 
 			if (log_context->minContinuityOverall > 0){
 				fprintf(fpratio, "continuity_max/min=%.5f\n",log_context->maxContinuityOverall / log_context->minContinuityOverall);
 			}else{
-				fprintf(fpratio, "continuity_max/min="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "continuity_max/min=%lld\n", INT64_MAX);
 			}
 			if (log_context->avgContinuityOverall > 0){
 				fprintf(fpratio, "continuity_max/avg=%.5f\n",log_context->maxContinuityOverall / log_context->avgContinuityOverall);
 			}else{
-				fprintf(fpratio, "continuity_max/avg="INT64_FORMAT"\n", INT64_MAX);
+				fprintf(fpratio, "continuity_max/avg=%lld\n", INT64_MAX);
 			}
 			fflush(fpratio);
 			fclose(fpratio);
@@ -3976,11 +3953,54 @@ static void cleanup_allocation_algorithm(
 }
 
 /*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,
+		udf_collector_context *context) {
+	if (node == NULL) {
+		return false;
+	}
+
+	if (IsA(node, Query)) {
+		return query_tree_walker((Query *) node, udf_collector_walker,
+				(void *) context,
+				QTW_EXAMINE_RTES);
+	}
+
+	/*For Aggref, we don't consider it as udf.*/
+
+	if(IsA(node,FuncExpr)){
+		if(!IsBuildInFunction(((FuncExpr *) node)->funcid)){
+			context->udf_exist = true;
+		}
+		return false;
+	}
+
+	return expression_tree_walker(node, udf_collector_walker,
+			(void *) context);
+
+	return false;
+}
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context) {
+
+	query_tree_walker(query, udf_collector_walker, (void *) context,
+	QTW_EXAMINE_RTES);
+
+	return;
+}
+
+
+/*
  * calculate_planner_segment_num
+ * fixedVsegNum is used by PBE, since all the execute should use the same number of vsegs.
  */
 SplitAllocResult *
 calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum) {
+		List *fullRangeTable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum) {
 	SplitAllocResult *result = NULL;
 	QueryResource *resource = NULL;
 	QueryResourceParameters *resource_parameters = NULL;
@@ -3989,14 +4009,14 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 	List *alloc_result = NIL;
 	split_to_segment_mapping_context context;
 
-	int planner_segments = -1; /*virtual segments number for explain statement */
+	int planner_segments = 0; /*virtual segments number for explain statement */
 
 	result = (SplitAllocResult *) palloc(sizeof(SplitAllocResult));
 	result->resource = NULL;
 	result->resource_parameters = NULL;
 	result->alloc_results = NIL;
 	result->relsType = NIL;
-	result->planner_segments = -1;
+	result->planner_segments = 0;
 	result->datalocalityInfo = makeStringInfo();
     result->datalocalityTime = 0;
 
@@ -4014,7 +4034,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		result->resource_parameters = NULL;
 		result->alloc_results = NIL;
 		result->relsType = NIL;
-		result->planner_segments = -1;
+		result->planner_segments = 0;
 		return result;
 	}
 
@@ -4039,6 +4059,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 		 * 5 data size of random "from" relation
 		 */
 
+		udf_collector_context udf_context;
+		udf_context.udf_exist = false;
+
+		find_udf(query, &udf_context);
+		isTableFunctionExists = udf_context.udf_exist;
 		/*convert range table list to oid list and check whether table function exists
 		 *we keep a full range table list and a range table list without result relation separately
 		 */
@@ -4065,7 +4090,15 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 		/*use inherit resource*/
 		if (resourceLife == QRL_INHERIT) {
-			resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+
+			if ( SPI_IsInPrepare() && (GetActiveQueryResource() == NULL) )
+			{
+				resource = NULL;
+			}
+			else
+			{
+				resource = AllocateResource(resourceLife, sliceNum, 0, 0, 0, NULL, 0);
+			}
 
 			saveQueryResourceParameters(
 							resource_parameters,  /* resource_parameters */
@@ -4216,6 +4249,11 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 				maxTargetSegmentNumber = enforce_virtual_segment_number;
 				minTargetSegmentNumber = enforce_virtual_segment_number;
 			}
+			/* in PBE mode, the execute should use the same vseg number. */
+			if(fixedVsegNum > 0 ){
+				maxTargetSegmentNumber = fixedVsegNum;
+				minTargetSegmentNumber = fixedVsegNum;
+			}
 			uint64_t before_rm_allocate_resource = gettime_microsec();
 
 			/* cost is use by RM to balance workload between hosts. the cost is at least one block size*/
@@ -4223,9 +4261,40 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 			mincost <<= 20;
 			int64 queryCost = context.total_size < mincost ? mincost : context.total_size;
 			if (QRL_NONE != resourceLife) {
-				resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
-						maxTargetSegmentNumber, minTargetSegmentNumber,
-						context.host_context.hostnameVolInfos, context.host_context.size);
+
+				if (SPI_IsInPrepare())
+				{
+					resource = NULL;
+					/*
+					 * prepare need to get resource quota from RM
+					 * and pass quota(planner_segments) to Orca or Planner to generate plan
+					 * the following executes(in PBE) should reallocate the same number
+					 * of resources.
+					 */
+					uint32 seg_num;
+					uint32 seg_num_min;
+					uint32 seg_memory_mb;
+					double seg_core;
+
+					GetResourceQuota(maxTargetSegmentNumber,
+					                 minTargetSegmentNumber,
+					                 &seg_num,
+					                 &seg_num_min,
+					                 &seg_memory_mb,
+					                 &seg_core);
+
+					planner_segments = seg_num;
+					minTargetSegmentNumber = planner_segments;
+					maxTargetSegmentNumber = planner_segments;
+				}
+				else
+				{
+					resource = AllocateResource(QRL_ONCE, sliceNum, queryCost,
+					                            maxTargetSegmentNumber,
+					                            minTargetSegmentNumber,
+					                            context.host_context.hostnameVolInfos,
+					                            context.host_context.size);
+				}
 
 				saveQueryResourceParameters(
 								resource_parameters,                   /* resource_parameters */
@@ -4258,7 +4327,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
 
 			if (resource == NULL) {
 				result->resource = NULL;
-				result->resource_parameters = NULL;
+				result->resource_parameters = resource_parameters;
 				result->alloc_results = NIL;
 				result->relsType = NIL;
 				result->planner_segments = planner_segments;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 07812aa..d0029fc 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -46,6 +46,7 @@
 #include "cdb/memquota.h"
 #include "executor/nodeFunctionscan.h"
 #include "nodes/stack.h"
+#include "cdb/cdbdatalocality.h"
 
 extern char *savedSeqServerHost;
 extern int savedSeqServerPort;
@@ -74,6 +75,7 @@ static int	_SPI_curid = -1;
 
 static PGconn *_QD_conn = NULL; /* To call back to the QD for SQL execution */
 static char *_QD_currently_prepared_stmt = NULL;
+static int SPI_prepare_depth = 0;
 
 static void _SPI_prepare_plan(const char *src, SPIPlanPtr plan);
 
@@ -106,6 +108,32 @@ static bool _SPI_checktuples(void);
 
 /* =================== interface functions =================== */
 
+bool SPI_IsInPrepare(void)
+{
+	if (SPI_prepare_depth > 0)
+	{
+		return true;
+	}
+	else if (SPI_prepare_depth < 0)
+	{
+		elog(ERROR, "Invalid SPI_prepare_depth %d while getting SPI prepare depth",
+		            SPI_prepare_depth);
+	}
+
+	return false;
+}
+
+void SPI_IncreasePrepareDepth(void)
+{
+	SPI_prepare_depth++;
+}
+
+void SPI_DecreasePrepareDepth(void)
+{
+	SPI_prepare_depth--;
+}
+
+
 int
 SPI_connect(void)
 {
@@ -566,6 +594,8 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 	_SPI_plan	plan;
 	_SPI_plan  *result;
 
+	SPI_IncreasePrepareDepth();
+
 	if (src == NULL || nargs < 0 || (nargs > 0 && argtypes == NULL))
 	{
 		SPI_result = SPI_ERROR_ARGUMENT;
@@ -591,9 +621,13 @@ SPI_prepare(const char *src, int nargs, Oid *argtypes)
 
 		/* copy plan to procedure context */
 		result = _SPI_copy_plan(&plan, _SPI_CPLAN_PROCXT);
+
+		SPI_DecreasePrepareDepth();
 	}
 	PG_CATCH();
 	{
+		SPI_DecreasePrepareDepth();
+
 		_SPI_end_call(true);
 		PG_RE_THROW();
 	}
@@ -1112,6 +1146,20 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
 	qtlist = copyObject(qtlist);
 	ptlist = copyObject(ptlist);
 
+	PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
+
+	if ( (Gp_role == GP_ROLE_DISPATCH) &&
+			 (stmt->resource_parameters != NULL) )
+	{
+		/*
+		 * Now, we want to allocate resource.
+		 */
+		stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
+				stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
+				stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
+				stmt->resource_parameters->vol_info_size);
+	}
+
 	/* If the plan has parameters, set them up */
 	if (spiplan->nargs > 0)
 	{
@@ -1812,22 +1860,21 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 				 * We only allocate resource for multiple executions of queries, NOT for utility commands.
 				 * SELECT/INSERT are supported at present.
 				 */
-				if( (queryTree->commandType == CMD_SELECT) ||
-				    (queryTree->commandType == CMD_INSERT) )
+				if((queryTree->commandType == CMD_SELECT) ||
+						(queryTree->commandType == CMD_INSERT))
 				{
-					if ( (Gp_role == GP_ROLE_DISPATCH) &&
-					     (stmt->resource == NULL) &&
-					     (stmt->resource_parameters != NULL) )
+					if ((Gp_role == GP_ROLE_DISPATCH) &&
+							(stmt->resource == NULL) &&
+							(stmt->resource_parameters != NULL))
 					{
 						stmt->resource = AllocateResource(stmt->resource_parameters->life,
-						                        stmt->resource_parameters->slice_size,
-						                        stmt->resource_parameters->iobytes,
-						                        stmt->resource_parameters->max_target_segment_num,
-						                        stmt->resource_parameters->min_target_segment_num,
-						                        stmt->resource_parameters->vol_info,
-						                        stmt->resource_parameters->vol_info_size);
+								stmt->resource_parameters->slice_size,
+								stmt->resource_parameters->iobytes,
+								stmt->resource_parameters->max_target_segment_num,
+								stmt->resource_parameters->min_target_segment_num,
+								stmt->resource_parameters->vol_info,
+								stmt->resource_parameters->vol_info_size);
 					}
-
 					originalStmt->resource = NULL;
 				}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/plan/planner.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 6b2e13e..ebad24f 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -91,6 +91,8 @@ planner_hook_type planner_hook = NULL;
 
 ParamListInfo PlannerBoundParamList = NULL;		/* current boundParams */
 
+static int PlanningDepth = 0;		/* Planning depth */
+
 /* Expression kind codes for preprocess_expression */
 #define EXPRKIND_QUAL			0
 #define EXPRKIND_TARGET			1
@@ -165,6 +167,32 @@ static void sort_canonical_gs_list(List *gs, int *p_nsets, Bitmapset ***p_sets);
 static Plan *pushdown_preliminary_limit(Plan *plan, Node *limitCount, int64 count_est, Node *limitOffset, int64 offset_est);
 bool is_dummy_plan(Plan *plan);
 
+
+bool is_in_planning_phase(void)
+{
+	if (PlanningDepth > 0)
+	{
+		return true;
+	}
+	else if (PlanningDepth < 0)
+	{
+		elog(ERROR, "Invalid PlanningDepth %d while getting planning phase", PlanningDepth);
+	}
+
+	return false;
+}
+
+void increase_planning_depth(void)
+{
+	PlanningDepth++;
+}
+
+void decrease_planning_depth(void)
+{
+	PlanningDepth--;
+}
+
+
 #ifdef USE_ORCA
 /**
  * Logging of optimization outcome
@@ -285,7 +313,7 @@ planner(Query *parse, int cursorOptions,
 	PlannedStmt *result = NULL;
 	instr_time	starttime, endtime;
 	ResourceNegotiatorResult *ppResult = (ResourceNegotiatorResult *) palloc(sizeof(ResourceNegotiatorResult));
-	SplitAllocResult initResult = {NULL, NULL, NIL, -1, NIL, NULL};
+	SplitAllocResult initResult = {NULL, NULL, NIL, 0, NIL, NULL};
 	ppResult->saResult = initResult;
 	ppResult->stmt = NULL;
 	static int plannerLevel = 0;
@@ -300,6 +328,8 @@ planner(Query *parse, int cursorOptions,
 	 * resource to run this query. After gaining the resource, we can perform the
 	 * actual optimization.
 	 */
+	increase_planning_depth();
+
 	plannerLevel++;
 	if (!resourceNegotiateDone)
 	{
@@ -309,6 +339,8 @@ planner(Query *parse, int cursorOptions,
       {
         resource_negotiator(parse, cursorOptions, boundParams, resourceLife, &ppResult);
 
+		decrease_planning_depth();
+
 		if(ppResult->stmt && ppResult->stmt->planTree)
 		{
 			isDispatchParallel = ppResult->stmt->planTree->dispatch == DISPATCH_PARALLEL;
@@ -318,6 +350,8 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  PG_CATCH();
 	  {
+		decrease_planning_depth();
+
 		if ((ppResult != NULL))
 		{
 		  pfree(ppResult);
@@ -430,7 +464,7 @@ planner(Query *parse, int cursorOptions,
 	  }
 	  else
 	  {
-	    gp_segments_for_planner = -1;
+	    gp_segments_for_planner = 0;
 	  }
 	  SetActiveQueryResource(savedQueryResource);
 	  if ((ppResult != NULL))
@@ -450,7 +484,7 @@ planner(Query *parse, int cursorOptions,
 		}
 		else
 		{
-			gp_segments_for_planner = -1;
+			gp_segments_for_planner = 0;
 		}
 		SetActiveQueryResource(savedQueryResource);
 
@@ -471,6 +505,7 @@ planner(Query *parse, int cursorOptions,
 	return result;
 }
 
+
 /*
  * The new framework for HAWQ 2.0 query optimizer
  */
@@ -481,6 +516,8 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
   PlannedStmt *plannedstmt = NULL;
   do
   {
+  		udf_collector_context udf_context;
+  		udf_context.udf_exist = false;
     SplitAllocResult *allocResult = NULL;
     Query *my_parse = copyObject(parse);
     ParamListInfo my_boundParams = copyParamList(boundParams);
@@ -496,12 +533,26 @@ resource_negotiator(Query *parse, int cursorOptions, ParamListInfo boundParams,
        */
       allocResult = calculate_planner_segment_num(my_parse, resourceLife,
                                           plannedstmt->rtable, plannedstmt->intoPolicy,
-                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1);
+                                          plannedstmt->nMotionNodes + plannedstmt->nInitPlans + 1,
+                                          -1);
 
       Assert(allocResult);
 
       (*result)->saResult = *allocResult;
       pfree(allocResult);
+    }else{
+    		find_udf(my_parse, &udf_context);
+    		if(udf_context.udf_exist){
+    			if ((resourceLife == QRL_ONCE) || (resourceLife == QRL_NONE)) {
+    				int64 mincost = min_cost_for_each_query;
+    				mincost <<= 20;
+    				int avgSliceNum = 3;
+    				(*result)->saResult.resource = AllocateResource(QRL_ONCE, avgSliceNum, mincost,
+    						GetUserDefinedFunctionVsegNum(),GetUserDefinedFunctionVsegNum(),NULL, 0);
+    			}else{
+    				(*result)->saResult.resource = AllocateResource(resourceLife, 0, 0, 0, 0, NULL, 0);
+    			}
+    		}
     }
   } while (0);
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/util/clauses.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index a80737f..a3d67f2 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -2831,8 +2831,15 @@ simplify_function(Oid funcid, Oid result_type, List *args,
 	if (!HeapTupleIsValid(func_tuple))
 		elog(ERROR, "cache lookup failed for function %u", funcid);
 
-	newexpr = evaluate_function(funcid, result_type, args,
-								func_tuple, context);
+	if (is_in_planning_phase())
+	{
+		newexpr = NULL;
+	}
+	else
+	{
+		newexpr = evaluate_function(funcid, result_type, args,
+		                            func_tuple, context);
+	}
 
 	if (large_const(newexpr, context->max_size))
 	{

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/backend/optimizer/util/walkers.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c
index 6f1ee51..a8222fb 100644
--- a/src/backend/optimizer/util/walkers.c
+++ b/src/backend/optimizer/util/walkers.c
@@ -162,6 +162,7 @@ expression_tree_walker(Node *node,
 		case T_PartBoundExpr:
 		case T_PartBoundInclusionExpr:
 		case T_PartBoundOpenExpr:
+		case T_RangeTblEntry	:
 			/* primitive node types with no expression subnodes */
 			break;
 		case T_Aggref:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/cdb/cdbdatalocality.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h
index d28fc3e..224cb53 100644
--- a/src/include/cdb/cdbdatalocality.h
+++ b/src/include/cdb/cdbdatalocality.h
@@ -50,6 +50,13 @@ typedef struct SplitAllocResult
 } SplitAllocResult;
 
 /*
+ * structure containing all relation range table entries.
+ */
+typedef struct udf_collector_context {
+	bool udf_exist;
+} udf_collector_context;
+
+/*
  * structure containing rel and type when execution
  */
 typedef struct CurrentRelType {
@@ -87,7 +94,17 @@ void saveQueryResourceParameters(
  * we calculate the appropriate planner segment_num.
  */
 SplitAllocResult * calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife,
-                                                List *rtable, GpPolicy *intoPolicy, int sliceNum);
+                                                List *rtable, GpPolicy *intoPolicy, int sliceNum, int fixedVsegNum);
+
+/*
+ * udf_collector_walker: the routine to file udfs.
+ */
+bool udf_collector_walker(Node *node,	udf_collector_context *context);
+
+/*
+ * find_udf: collect all udf, and store them into the udf_collector_context.
+ */
+void find_udf(Query *query, udf_collector_context *context);
 
 FILE *fp;
 FILE *fpaoseg;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/executor/spi.h
----------------------------------------------------------------------
diff --git a/src/include/executor/spi.h b/src/include/executor/spi.h
index 867daf5..7b4ba1b 100644
--- a/src/include/executor/spi.h
+++ b/src/include/executor/spi.h
@@ -155,4 +155,11 @@ extern uint64 SPI_GetMemoryReservation(void);
 extern void SPI_ReserveMemory(uint64 mem_reserved);
 extern bool SPI_IsMemoryReserved(void);
 
+/**
+ * Query resource related routines.
+ */
+extern bool SPI_IsInPrepare(void);
+extern void SPI_IncreasePrepareCounter(void);
+extern void SPI_DecreasePrepareCounter(void);
+
 #endif   /* SPI_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4d997b57/src/include/optimizer/planner.h
----------------------------------------------------------------------
diff --git a/src/include/optimizer/planner.h b/src/include/optimizer/planner.h
index 48de348..cc2eb7d 100644
--- a/src/include/optimizer/planner.h
+++ b/src/include/optimizer/planner.h
@@ -50,4 +50,8 @@ extern bool choose_hashed_grouping(PlannerInfo *root,
 								   double dNumGroups, 
 								   AggClauseCounts *agg_counts);
 
+extern bool is_in_planning_phase(void);
+extern void increase_planning_depth(void);
+extern void decrease_planning_depth(void);
+
 #endif   /* PLANNER_H */


[08/50] incubator-hawq git commit: HAWQ-473. Implement adding a entry into gp_configuration_history and add a description column to show the reason of this segment is down

Posted by sh...@apache.org.
HAWQ-473. Implement adding a entry into gp_configuration_history and
    	  add a description column to show the reason of this segment is down


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d1780628
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d1780628
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d1780628

Branch: refs/heads/HAWQ-459
Commit: d178062876f0c55db7b1ac74946bd2ba096b779f
Parents: 2a15066
Author: Wen Lin <wl...@pivotal.io>
Authored: Tue Mar 15 10:00:16 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Mar 15 10:00:16 2016 +0800

----------------------------------------------------------------------
 .../communication/rmcomm_RM2RMSEG.c             |  28 +-
 .../resourcemanager/include/resourcepool.h      |  43 +-
 src/backend/resourcemanager/requesthandler.c    | 109 ++-
 .../resourcemanager/requesthandler_RMSEG.c      |   1 -
 .../resourcebroker/resourcebroker_LIBYARN.c     |  82 +-
 .../resourcebroker_LIBYARN_proc.c               |   1 -
 src/backend/resourcemanager/resourcemanager.c   |  91 +-
 src/backend/resourcemanager/resourcepool.c      | 857 ++++++++++++-------
 src/backend/resourcemanager/resqueuemanager.c   |   3 +-
 src/backend/utils/gp/segadmin.c                 |   3 +-
 src/include/catalog/gp_configuration.h          |  27 +-
 src/include/catalog/gp_segment_config.h         |  11 +-
 src/include/catalog/pg_proc.h                   |   5 +-
 src/include/utils/builtins.h                    |   1 +
 tools/bin/gppylib/data/2.0.json                 |  32 +-
 15 files changed, 823 insertions(+), 471 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index fe4a595..ca332c9 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -231,11 +231,23 @@ void receivedRUAliveResponse(AsyncCommMessageHandlerContext  context,
 			 * This call makes resource pool remove unused containers.
 			 */
 			returnAllGRMResourceFromSegment(segres);
+
+			segres->Stat->StatusDesc |= SEG_STATUS_FAILED_PROBING_SEGMENT;
 			/* Set the host down in gp_segment_configuration table */
 			if (Gp_role != GP_ROLE_UTILITY)
 			{
+				SimpStringPtr description = build_segment_status_description(segres->Stat);
 				update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
-									  SEGMENT_STATUS_DOWN);
+									  SEGMENT_STATUS_DOWN,
+									  (description->Len > 0)?description->Str:"");
+				add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+										GET_SEGRESOURCE_HOSTNAME(segres),
+										description->Str);
+				if (description != NULL)
+				{
+					freeSimpleStringContent(description);
+					rm_pfree(PCONTEXT, description);
+				}
 			}
 			/* Set the host down. */
 			elog(WARNING, "Resource manager sets host %s from up to down "
@@ -281,10 +293,22 @@ void sentRUAliveError(AsyncCommMessageHandlerContext context)
 		 * This call makes resource pool remove unused containers.
 		 */
 		returnAllGRMResourceFromSegment(segres);
+		segres->Stat->StatusDesc |= SEG_STATUS_COMMUNICATION_ERROR;
 		/* Set the host down in gp_segment_configuration table */
 		if (Gp_role != GP_ROLE_UTILITY)
 		{
-			update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET, SEGMENT_STATUS_DOWN);
+			SimpStringPtr description = build_segment_status_description(segres->Stat);
+			update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+								  SEGMENT_STATUS_DOWN,
+								  (description->Len > 0)?description->Str:"");
+			add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+									GET_SEGRESOURCE_HOSTNAME(segres),
+									description->Str);
+			if (description != NULL)
+			{
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
+			}
 		}
 		/* Set the host down. */
 		elog(LOG, "Resource manager sets host %s from up to down "

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index 6e1412c..5feedd3 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -145,13 +145,14 @@ struct SegStatData {
 	int32_t			ID;					/* Internal ID.						  */
 	uint16_t		FailedTmpDirNum;	/* Failed temporary directory number */
 	uint8_t			FTSAvailable;		/* If it is available now.			  */
-	uint8_t			GRMAvailable;		/* If it is global resource available.*/
-
+	uint8_t			GRMHandled;			/* If its GRM status is handled */
 	uint32_t		FTSTotalMemoryMB;		/* FTS reports memory capacity.   */
 	uint32_t		FTSTotalCore;			/* FTS reports core capacity.	  */
 	uint32_t		GRMTotalMemoryMB;		/* GRM reports memory capacity.	  */
 	uint32_t		GRMTotalCore;			/* GRM reports core capacity. 	  */
 	uint64_t		RMStartTimestamp;		/* RM process reset timestamp */
+	uint32_t		StatusDesc;				/* Description of status */
+	uint32_t		Reserved;
 	SegInfoData		Info;					/* 64-bit aligned.				  */
 };
 
@@ -168,12 +169,9 @@ enum SegAvailabilityStatus {
 };
 
 int setSegStatHAWQAvailability( SegStat machine, uint8_t newstatus);
-int setSegStatGLOBAvailability( SegStat machine, uint8_t newstatus);
 
 #define IS_SEGSTAT_FTSAVAILABLE(seg) \
 		((seg)->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
-#define IS_SEGSTAT_GRMAVAILABLE(seg) \
-	    ((seg)->GRMAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
 
 /* Generate SegStat instance's report as a string saved in self maintained buffer. */
 void  generateSegStatReport(SegStat segstat, SelfMaintainBuffer buff);
@@ -610,7 +608,7 @@ int returnResourceToResourcePool(int 		memory,
 void returnAllGRMResourceFromSegment(SegResource segres);
 void dropAllGRMContainersFromSegment(SegResource segres);
 
-void returnAllGRMResourceFromGRMUnavailableSegments(void);
+void returnAllGRMResourceFromUnavailableSegments(void);
 
 void generateSegResourceReport(int32_t nodeid, SelfMaintainBuffer buff);
 
@@ -632,7 +630,7 @@ int addOrderedResourceAvailTreeIndexByRatio(uint32_t ratio, BBST *tree);
 int getOrderedResourceAvailTreeIndexByRatio(uint32_t ratio, BBST *tree);
 int getOrderedResourceAllocTreeIndexByRatio(uint32_t ratio, BBST *tree);
 
-void setAllSegResourceGRMUnavailable(void);
+void setAllSegResourceGRMUnhandled(void);
 
 void resetAllSegmentsGRMContainerFailAllocCount(void);
 
@@ -671,13 +669,11 @@ void adjustMemoryCoreValue(uint32_t *memorymb, uint32_t *core);
 
 /* Clean up gp_segment_configuration */
 void cleanup_segment_config(void);
+
+#define SEG_STATUS_DESCRIPTION_UP "segment is up"
 /* update a segment's status in gp_segment_configuration table */
-void update_segment_status(int32_t id, char status);
-/* update a segment's status and failed temporary directory
- * in gp_segment_configuration table
- */
-void update_segment_failed_tmpdir
-(int32_t id, char status, int32_t failedNum, char* failedTmpDir);
+void update_segment_status(int32_t id, char status, char* description);
+
 /* Add a new entry into gp_segment_configuration table*/
 void add_segment_config_row(int32_t 	 id,
 							char		*hostname,
@@ -685,8 +681,25 @@ void add_segment_config_row(int32_t 	 id,
 							uint32_t 	 port,
 							char 		 role,
 							char		 status,
-							uint32_t	 failed_tmpdir_num,
-							char*		 failed_tmpdir);
+							char*		 description);
+
+/*
+ * SegStatData's StatusDesc is a combination of below flags
+ */
+#define	SEG_STATUS_HEARTBEAT_TIMEOUT			0x00000001
+#define	SEG_STATUS_FAILED_PROBING_SEGMENT		0x00000002
+#define	SEG_STATUS_COMMUNICATION_ERROR			0x00000004
+#define	SEG_STATUS_FAILED_TMPDIR				0x00000008
+#define	SEG_STATUS_RM_RESET						0x00000010
+#define	SEG_STATUS_NO_GRM_NODE_REPORT			0x00000020
+
+/* Add a new entry into gp_configuration_history table */
+void add_segment_history_row(int32_t id,
+							 char* hostname,
+							 char* description);
+
+/* build a string of status description based on SegStat */
+SimpStringPtr build_segment_status_description(SegStat segstat);
 
 /*
  * In resource pool, segment's id starts from 0, however in gp_segment_configuration table,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/requesthandler.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler.c b/src/backend/resourcemanager/requesthandler.c
index ba35e2e..a06d169 100644
--- a/src/backend/resourcemanager/requesthandler.c
+++ b/src/backend/resourcemanager/requesthandler.c
@@ -746,29 +746,13 @@ bool handleRMSEGRequestIMAlive(void **arg)
 	destroySelfMaintainBuffer(&newseginfo);
 
 	newsegstat->ID 				 = SEGSTAT_ID_INVALID;
-	newsegstat->GRMAvailable 	 = RESOURCE_SEG_STATUS_UNSET;
 
 	RPCRequestHeadIMAlive header = SMBUFF_HEAD(RPCRequestHeadIMAlive,
 												&(conntrack->MessageBuff));
 	newsegstat->FailedTmpDirNum  = header->TmpDirBrokenCount;
 	newsegstat->RMStartTimestamp = header->RMStartTimestamp;
-
-	/*
-	 * Check if the there is any failed temporary directory on this segment.
-	 * if has, master considers this segment as down, even it has heart-beat report.
-	 */
-	if (newsegstat->FailedTmpDirNum == 0)
-	{
-		newsegstat->FTSAvailable = RESOURCE_SEG_STATUS_AVAILABLE;
-	}
-	else
-	{
-		elog(RMLOG, "Resource manager finds there is %d failed temporary directories "
-					"on this segment, "
-					"so mark this segment unavailable.",
-					newsegstat->FailedTmpDirNum);
-		newsegstat->FTSAvailable = RESOURCE_SEG_STATUS_UNAVAILABLE;
-	}
+	newsegstat->StatusDesc = 0;
+	newsegstat->Reserved   = 0;
 
 	bool capstatchanged = false;
 	if ( addHAWQSegWithSegStat(newsegstat, &capstatchanged) != FUNC_RETURN_OK )
@@ -977,13 +961,13 @@ bool handleRMRequestSegmentIsDown(void **arg)
 
 	while( (hostname - SMBUFF_CONTENT(&(conntrack->MessageBuff)) <
 			getSMBContentSize(&(conntrack->MessageBuff))) &&
-		   *hostname != '\0' )
+			*hostname != '\0' )
 	{
 		hostnamelen = strlen(hostname);
 		res = getSegIDByHostName(hostname, hostnamelen, &segid);
 		if ( res == FUNC_RETURN_OK )
 		{
-			/* Get resourceinfo of the expected host. */
+			/* Get resource info of the expected host. */
 			SegResource segres = getSegResource(segid);
 			Assert( segres != NULL );
 
@@ -1002,45 +986,56 @@ bool handleRMRequestSegmentIsDown(void **arg)
 			else
 			{
 				elog(RMLOG, "Resource manager probes the status of host %s by "
-						 	"sending RUAlive request.",
+							"sending RUAlive request.",
 							hostname);
 
-		        res = sendRUAlive(hostname);
-		        /* IN THIS CASE, the segment is considered as down. */
-		        if (res != FUNC_RETURN_OK)
-		        {
-		        	/*----------------------------------------------------------
-		        	 * This call makes resource manager able to adjust queue and
-		        	 * mem/core trackers' capacity.
-		        	 *----------------------------------------------------------
-		        	 */
-		        	setSegResHAWQAvailability(segres,
-		        							  RESOURCE_SEG_STATUS_UNAVAILABLE);
-
-		        	/* Make resource pool remove unused containers */
-		        	returnAllGRMResourceFromSegment(segres);
-		        	/* Set the host down in gp_segment_configuration table */
-		        	if (Gp_role != GP_ROLE_UTILITY)
-		        	{
-		        		update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
-		        							  SEGMENT_STATUS_DOWN);
-		        	}
-
-		        	/* Set the host down. */
-		        	elog(LOG, "Resource manager sets host %s from up to down "
-		        			  "due to not reaching host.", hostname);
-		        }
-		        else
-		        {
-		        	elog(RMLOG, "Resource manager triggered RUAlive request to "
-		        				"host %s.",
+				res = sendRUAlive(hostname);
+				/* IN THIS CASE, the segment is considered as down. */
+				if (res != FUNC_RETURN_OK)
+				{
+					/*----------------------------------------------------------
+					 * This call makes resource manager able to adjust queue and
+					 * mem/core trackers' capacity.
+					 *----------------------------------------------------------
+					 */
+					setSegResHAWQAvailability(segres,
+											  RESOURCE_SEG_STATUS_UNAVAILABLE);
+
+					/* Make resource pool remove unused containers */
+					returnAllGRMResourceFromSegment(segres);
+					/* Set the host down in gp_segment_configuration table */
+					segres->Stat->StatusDesc |= SEG_STATUS_FAILED_PROBING_SEGMENT;
+					if (Gp_role != GP_ROLE_UTILITY)
+					{
+						SimpStringPtr description = build_segment_status_description(segres->Stat);
+						update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+											  SEGMENT_STATUS_DOWN,
+											  (description->Len > 0)?description->Str:"");
+						add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+												hostname,
+												description->Str);
+						if (description != NULL)
+						{
+							freeSimpleStringContent(description);
+							rm_pfree(PCONTEXT, description);
+						}
+					}
+
+					/* Set the host down. */
+					elog(LOG, "Resource manager sets host %s from up to down "
+							  "due to not reaching host.", hostname);
+				}
+				else
+				{
+					elog(RMLOG, "Resource manager triggered RUAlive request to "
+								"host %s.",
 								hostname);
-		        }
+				}
 			}
 		}
 		else {
 			elog(WARNING, "Resource manager cannot find host %s to check status, "
-					  	  "skip it.",
+						  "skip it.",
 						  hostname);
 		}
 
@@ -1124,7 +1119,7 @@ bool handleRMRequestTmpDir(void **arg)
                                    RESPONSE_QD_TMPDIR);
         
         elog(LOG, "Resource manager assigned temporary directory %s",
-        		  tmpdir->Str);
+                  tmpdir->Str);
     }
 
     conntrack->ResponseSent = false;
@@ -1271,11 +1266,11 @@ bool handleRMRequestDummy(void **arg)
                                sizeof(response),
                                conntrack->MessageMark1,
                                conntrack->MessageMark2,
-							   RESPONSE_DUMMY);
+                               RESPONSE_DUMMY);
     conntrack->ResponseSent = false;
-	MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
-	PCONTRACK->ConnToSend = lappend(PCONTRACK->ConnToSend, conntrack);
-	MEMORY_CONTEXT_SWITCH_BACK
+    MEMORY_CONTEXT_SWITCH_TO(PCONTEXT)
+    PCONTRACK->ConnToSend = lappend(PCONTRACK->ConnToSend, conntrack);
+    MEMORY_CONTEXT_SWITCH_BACK
 
     return true;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/requesthandler_RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/requesthandler_RMSEG.c b/src/backend/resourcemanager/requesthandler_RMSEG.c
index e667c1a..90946a0 100644
--- a/src/backend/resourcemanager/requesthandler_RMSEG.c
+++ b/src/backend/resourcemanager/requesthandler_RMSEG.c
@@ -182,7 +182,6 @@ int refreshLocalHostInstance(void)
 		RMSEG_INBUILDHOST->GRMTotalCore     = 0.0;
 		RMSEG_INBUILDHOST->FTSTotalMemoryMB = DRMGlobalInstance->SegmentMemoryMB;
 		RMSEG_INBUILDHOST->FTSTotalCore     = DRMGlobalInstance->SegmentCore;
-		RMSEG_INBUILDHOST->GRMAvailable    = RESOURCE_SEG_STATUS_UNSET;
 		RMSEG_INBUILDHOST->FTSAvailable    = RESOURCE_SEG_STATUS_AVAILABLE;
 		RMSEG_INBUILDHOST->ID				= SEGSTAT_ID_INVALID;
 		RMSEG_INBUILDHOST->FailedTmpDirNum  = failedTmpDirNum;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index 1e8c7ac..d446a6d 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -536,7 +536,9 @@ int handleRB2RM_ClusterReport(void)
 	uint32_t	segsize;
 	int			fd 		    = ResBrokerNotifyPipe[0];
 	int			piperes     = 0;
-	List	   *segstats	= NULL;
+	List		*segstats	= NULL;
+	List		*allsegres  = NULL;
+	ListCell	*cell		= NULL;
 
 	PRESPOOL->RBClusterReportCounter++;
 
@@ -642,11 +644,7 @@ int handleRB2RM_ClusterReport(void)
 		return res;
 	}
 
-	/*
-	 * Set all current segments GRM unavailable, only the segments identified by
-	 * one segment in segstats are available.
-	 */
-	setAllSegResourceGRMUnavailable();
+	setAllSegResourceGRMUnhandled();
 
 	/*
 	 * Start to update resource pool content. The YARN cluster total size is
@@ -683,6 +681,66 @@ int handleRB2RM_ClusterReport(void)
 		rm_pfree(PCONTEXT, segstat);
 		segstats = list_delete_first(segstats);
 	}
+
+	/*
+	 * iterate all segments without GRM report,
+	 * and update its status.
+	 */
+	getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres);
+	foreach(cell, allsegres)
+	{
+		SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
+		bool statusDescChange = false;
+
+		/*
+		 * skip segments handled in GRM report list
+		 */
+		if (segres->Stat->GRMHandled)
+			continue;
+
+		/*
+		 * Set no GRM node report flag for this segment.
+		 */
+		if ((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT) == 0)
+		{
+			segres->Stat->StatusDesc |= SEG_STATUS_NO_GRM_NODE_REPORT;
+			statusDescChange = true;
+		}
+
+		if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
+		{
+			/*
+			 * This segment is FTS available, but master hasn't
+			 * gotten its GRM node report, so set this segment to DOWN.
+			 */
+			setSegResHAWQAvailability(segres, RESOURCE_SEG_STATUS_UNAVAILABLE);
+		}
+
+		Assert(!IS_SEGSTAT_FTSAVAILABLE(segres->Stat));
+		if (statusDescChange && Gp_role != GP_ROLE_UTILITY)
+		{
+			SimpStringPtr description = build_segment_status_description(segres->Stat);
+			update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+									SEGMENT_STATUS_DOWN,
+									 (description->Len > 0)?description->Str:"");
+			add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+									GET_SEGRESOURCE_HOSTNAME(segres),
+									description->Str);
+
+			elog(LOG, "Resource manager hasn't gotten GRM node report for segment(%s),"
+						"updates its status:'%c', description:%s",
+						GET_SEGRESOURCE_HOSTNAME(segres),
+						SEGMENT_STATUS_DOWN,
+						(description->Len > 0)?description->Str:"");
+			if (description != NULL)
+			{
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
+			}
+		}
+	}
+	freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
+
 	MEMORY_CONTEXT_SWITCH_BACK
 
 	elog(LOG, "Resource manager YARN resource broker counted HAWQ cluster now "
@@ -694,10 +752,10 @@ int handleRB2RM_ClusterReport(void)
 			  PRESPOOL->GRMTotalHavingNoHAWQNode.Core);
 
 	/*
-	 * If the segment is not GRM available, RM should return all containers
-	 * located upon them.
+	 * If the segment is GRM unavailable or FTS unavailable,
+	 * RM should return all containers located upon them.
 	 */
-	returnAllGRMResourceFromGRMUnavailableSegments();
+	returnAllGRMResourceFromUnavailableSegments();
 
 	/* Refresh available node count. */
 	refreshAvailableNodeCount();
@@ -706,7 +764,7 @@ int handleRB2RM_ClusterReport(void)
 	PQUEMGR->GRMQueueCapacity	 	= response.QueueCapacity;
 	PQUEMGR->GRMQueueCurCapacity 	= response.QueueCurCapacity;
 	PQUEMGR->GRMQueueMaxCapacity 	= (response.QueueMaxCapacity > 0 &&
-								   	   response.QueueMaxCapacity <= 1) ?
+									  response.QueueMaxCapacity <= 1) ?
 									  response.QueueMaxCapacity :
 									  PQUEMGR->GRMQueueMaxCapacity;
 	PQUEMGR->GRMQueueResourceTight 	= response.ResourceTight > 0 ? true : false;
@@ -714,9 +772,9 @@ int handleRB2RM_ClusterReport(void)
 	refreshResourceQueueCapacity(false);
 	refreshActualMinGRMContainerPerSeg();
 
-    PRESPOOL->LastUpdateTime = gettime_microsec();
+	PRESPOOL->LastUpdateTime = gettime_microsec();
 
-    return FUNC_RETURN_OK;
+	return FUNC_RETURN_OK;
 }
 
 /*

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
index 3afae7b..f99ed48 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN_proc.c
@@ -1482,7 +1482,6 @@ int RB2YARN_getClusterReport(DQueue hosts)
 
     		segstat->ID     					 = SEGSTAT_ID_INVALID;
     		segstat->FTSAvailable 				 = RESOURCE_SEG_STATUS_UNSET;
-    		segstat->GRMAvailable 				 = RESOURCE_SEG_STATUS_AVAILABLE;
     		segstat->GRMTotalMemoryMB  	   		 = pnodereport->memoryCapability;
     		segstat->GRMTotalCore	  	   		 = pnodereport->vcoresCapability;
     		segstat->FTSTotalMemoryMB	   		 = 0;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 9a4bb85..138c5a0 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -489,7 +489,6 @@ int ResManagerMainServer2ndPhase(void)
                            PostPortNumber,
                            SEGMENT_ROLE_MASTER_CONFIG,
                            SEGMENT_STATUS_UP,
-                           0,
                            "");
 
 	/* Load queue and user definition as no DDL now. */
@@ -2096,12 +2095,9 @@ int generateAllocRequestToBroker(void)
 		/*
 		 * Resource manager skips this segment if
 		 * 1) Not FTS available;
-		 * 2) Not GRM available;
-		 * 3) Having resource decrease pending.
+		 * 2) Having resource decrease pending.
 		 */
 		if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
-			(DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
-			 !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
 			(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
 		{
 			continue;
@@ -2311,12 +2307,9 @@ void completeAllocRequestToBroker(int32_t 	 *reqmem,
 		/*
 		 * Resource manager skips this segment if
 		 * 1) Not FTS available;
-		 * 2) Not GRM available;
-		 * 3) Having resource decrease pending.
+		 * 2) Having resource decrease pending.
 		 */
-		if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
-			(DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
-			 !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
+		if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat)  ||
 			(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
 		{
 			index++;
@@ -2391,12 +2384,9 @@ void completeAllocRequestToBroker(int32_t 	 *reqmem,
 			/*
 			 * Resource manager skips this segment if
 			 * 1) Not FTS available;
-			 * 2) Not GRM available;
-			 * 3) Having resource decrease pending.
+			 * 2) Having resource decrease pending.
 			 */
 			if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
-				(DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
-				 !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
 				(segres->DecPending.MemoryMB > 0 && segres->DecPending.Core > 0))
 			{
 				index++;
@@ -2566,31 +2556,51 @@ void updateStatusOfAllNodes()
 	curtime = gettime_microsec();
 	for(uint32_t idx = 0; idx < PRESPOOL->SegmentIDCounter; idx++)
 	{
-	    node = getSegResource(idx);
-        if (node != NULL &&
-            (curtime - node->LastUpdateTime >
+		node = getSegResource(idx);
+		uint8_t oldStatus = node->Stat->FTSAvailable;
+		if (node != NULL &&
+			 (curtime - node->LastUpdateTime >
 			 1000000LL * rm_segment_heartbeat_timeout) &&
-			IS_SEGSTAT_FTSAVAILABLE(node->Stat) )
-        {
-        	/*
-        	 * This call makes resource manager able to adjust queue and mem/core
-        	 * trackers' capacity.
-        	 */
-        	setSegResHAWQAvailability(node, RESOURCE_SEG_STATUS_UNAVAILABLE);
-        	/*
-        	 * This call makes resource pool remove unused containers.
-        	 */
-        	returnAllGRMResourceFromSegment(node);
-        	if (Gp_role != GP_ROLE_UTILITY)
-        	{
-        		update_segment_status(idx + REGISTRATION_ORDER_OFFSET, SEGMENT_STATUS_DOWN);
-        	}
+			 (node->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0)
+		{
+			/*
+			 * This segment is heartbeat timeout, update its description
+			 * and set it to unavailable if needed.
+			 */
+			if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+			{
+				/*
+				 * This call makes resource manager able to adjust queue and mem/core
+				 * trackers' capacity.
+				 */
+				setSegResHAWQAvailability(node, RESOURCE_SEG_STATUS_UNAVAILABLE);
+				/*
+				 * This call makes resource pool remove unused containers.
+				 */
+				returnAllGRMResourceFromSegment(node);
+				changedstatus = true;
+			}
 
-        	elog(WARNING, "Resource manager sets host %s from up to down.",
-        			  	  GET_SEGRESOURCE_HOSTNAME(node));
+			node->Stat->StatusDesc |= SEG_STATUS_HEARTBEAT_TIMEOUT;
+			if (Gp_role != GP_ROLE_UTILITY)
+			{
+				SimpStringPtr description = build_segment_status_description(node->Stat);
+				update_segment_status(idx + REGISTRATION_ORDER_OFFSET,
+										SEGMENT_STATUS_DOWN,
+										(description->Len > 0)?description->Str:"");
+				add_segment_history_row(idx + REGISTRATION_ORDER_OFFSET,
+										GET_SEGRESOURCE_HOSTNAME(node),
+										(description->Len > 0)?description->Str:"");
+				if (description != NULL)
+				{
+					freeSimpleStringContent(description);
+					rm_pfree(PCONTEXT, description);
+				}
+			}
 
-        	changedstatus = true;
-        }
+			elog(WARNING, "Resource manager sets host %s heartbeat timeout.",
+						  GET_SEGRESOURCE_HOSTNAME(node));
+		}
 	}
 
 	if ( changedstatus )
@@ -2711,11 +2721,10 @@ int  loadHostInformationIntoResourcePool(void)
 
         /* Build machine info instance. */
         SegStat segstat = (SegStat)rm_palloc0(PCONTEXT,
-                                   	   	   	  offsetof(SegStatData, Info) +
-											  seginfobuff.Cursor + 1);
-        segstat->ID      		   = SEGSTAT_ID_INVALID;
-        segstat->GRMAvailable     = RESOURCE_SEG_STATUS_UNSET;
-        segstat->FTSAvailable     = RESOURCE_SEG_STATUS_AVAILABLE;
+                                              offsetof(SegStatData, Info) +
+                                              seginfobuff.Cursor + 1);
+        segstat->ID                = SEGSTAT_ID_INVALID;
+        segstat->FTSAvailable      = RESOURCE_SEG_STATUS_AVAILABLE;
         segstat->FTSTotalMemoryMB  = DRMGlobalInstance->SegmentMemoryMB;
         segstat->FTSTotalCore      = DRMGlobalInstance->SegmentCore;
         segstat->GRMTotalMemoryMB  = 0;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index 329a5c6..6a36801 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -26,6 +26,8 @@
 #include "resourcepool.h"
 #include "gp-libpq-fe.h"
 #include "gp-libpq-int.h"
+#include "utils/builtins.h"
+#include "catalog/pg_proc.h"
 
 void addSegResourceAvailIndex(SegResource segres);
 void addSegResourceAllocIndex(SegResource segres);
@@ -467,12 +469,99 @@ cleanup:
 		PQclear(result);
 	PQfinish(conn);
 }
+
+/*
+ * Remove all entries in gp_configuration_history.
+ *
+ * gp_remove_segment_history()
+ *
+ * Returns:
+ *   true upon success, otherwise throws error.
+ */
+Datum
+gp_remove_segment_history(PG_FUNCTION_ARGS)
+{
+	int	libpqres = CONNECTION_OK;
+	PGconn *conn = NULL;
+	char conninfo[512];
+	PQExpBuffer sql = NULL;
+	PGresult* result = NULL;
+	int ret = true;
+
+	if (!superuser())
+		elog(ERROR, "gp_remove_segment_history can only be run by a superuser");
+
+	if (Gp_role != GP_ROLE_UTILITY)
+		elog(ERROR, "gp_remove_segment_history can only be run in utility mode");
+
+	sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
+			"dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
+	conn = PQconnectdb(conninfo);
+	if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
+	{
+		elog(WARNING, "Fail to connect database when cleanup "
+					  "segment configuration catalog table, error code: %d, %s",
+					  libpqres,
+					  PQerrorMessage(conn));
+		PQfinish(conn);
+		PG_RETURN_BOOL(false);
+	}
+
+	result = PQexec(conn, "BEGIN");
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+					  "segment history catalog table, reason : %s",
+					  "BEGIN",
+					  PQresultErrorMessage(result));
+		ret = false;
+		goto cleanup;
+	}
+	PQclear(result);
+
+	sql = createPQExpBuffer();
+	appendPQExpBuffer(sql,"DELETE FROM gp_configuration_history");
+	result = PQexec(conn, sql->data);
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+					  "segment history catalog table, reason : %s",
+					  sql->data,
+					  PQresultErrorMessage(result));
+		ret = false;
+		goto cleanup;
+	}
+	PQclear(result);
+
+	result = PQexec(conn, "COMMIT");
+	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		elog(WARNING, "Fail to run SQL: %s when cleanup "
+					  "segment history catalog table, reason : %s",
+					  "COMMIT",
+					  PQresultErrorMessage(result));
+		ret = false;
+		goto cleanup;
+	}
+
+	elog(LOG, "Cleanup segment history catalog table successfully!");
+
+cleanup:
+	if(sql)
+		destroyPQExpBuffer(sql);
+	if(result)
+		PQclear(result);
+	PQfinish(conn);
+
+	PG_RETURN_BOOL(ret);
+}
+
 /*
  *  update a segment's status in gp_segment_configuration table.
  *  id : registration order of this segment
  *  status : new status of this segment
  */
-void update_segment_status(int32_t id, char status)
+void update_segment_status(int32_t id, char status, char* description)
 {
 	int	libpqres = CONNECTION_OK;
 	PGconn *conn = NULL;
@@ -480,15 +569,22 @@ void update_segment_status(int32_t id, char status)
 	PQExpBuffer sql = NULL;
 	PGresult* result = NULL;
 
+	if (status == SEGMENT_STATUS_UP)
+		Assert(strlen(description) == 0);
+	else if (status == SEGMENT_STATUS_DOWN)
+		Assert(strlen(description) != 0);
+	else
+		Assert(0);
+
 	sprintf(conninfo, "options='-c gp_session_role=UTILITY -c allow_system_table_mods=dml' "
 			"dbname=template1 port=%d connect_timeout=%d", master_addr_port, CONNECT_TIMEOUT);
 	conn = PQconnectdb(conninfo);
 	if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
 	{
 		elog(WARNING, "Fail to connect database when update segment's status "
-				      "in segment configuration catalog table, error code: %d, %s",
-				      libpqres,
-				      PQerrorMessage(conn));
+					  "in segment configuration catalog table, error code: %d, %s",
+					  libpqres,
+					  PQerrorMessage(conn));
 		PQfinish(conn);
 		return;
 	}
@@ -497,23 +593,24 @@ void update_segment_status(int32_t id, char status)
 	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
 	{
 		elog(WARNING, "Fail to run SQL: %s when update segment's status "
-				      "in segment configuration catalog table, reason : %s",
-				      "BEGIN",
-				      PQresultErrorMessage(result));
+					  "in segment configuration catalog table, reason : %s",
+					  "BEGIN",
+					  PQresultErrorMessage(result));
 		goto cleanup;
 	}
 	PQclear(result);
 
 	sql = createPQExpBuffer();
-	appendPQExpBuffer(sql,"UPDATE gp_segment_configuration SET status='%c' WHERE registration_order=%d",
-						   status, id);
+	appendPQExpBuffer(sql, "UPDATE gp_segment_configuration SET status='%c', "
+						   "description='%s' WHERE registration_order=%d",
+						   status, description, id);
 	result = PQexec(conn, sql->data);
 	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
 	{
 		elog(WARNING, "Fail to run SQL: %s when update segment's status "
-				      "in segment configuration catalog table, reason : %s",
-				      sql->data,
-				      PQresultErrorMessage(result));
+					  "in segment configuration catalog table, reason : %s",
+					  sql->data,
+					  PQresultErrorMessage(result));
 		goto cleanup;
 	}
 	PQclear(result);
@@ -528,9 +625,10 @@ void update_segment_status(int32_t id, char status)
 		goto cleanup;
 	}
 
-	elog(LOG, "Update a segment's status to '%c' in segment configuration catalog table,"
-			  "registration_order : %d",
-			  status, id);
+	elog(LOG, "Update a segment(registration_order:%d)'s status to '%c',"
+			  "description to '%s', "
+			  "in segment configuration catalog table,",
+			  id, status, description);
 
 cleanup:
 	if(sql)
@@ -541,19 +639,17 @@ cleanup:
 }
 
 /*
- *  update a segment's status and failed tmp dir
- *  in gp_segment_configuration table.
+ *  Insert a row into gp_configuration_history,
+ *  to record the status change of a segment.
  *  id : registration order of this segment
- *  status : new status of this segment
- *  failedNum : number of failed temporary directory
- *  failedTmpDir : failed temporary directory list, separated by comma
+ *  hostname : hostname of this segment
+ *  description : description of status change
  */
-void update_segment_failed_tmpdir
-(int32_t id, char status, int32_t failedNum, char* failedTmpDir)
+void add_segment_history_row(int32_t id, char* hostname, char* description)
 {
 	int	libpqres = CONNECTION_OK;
 	PGconn *conn = NULL;
-	char conninfo[512];
+	char conninfo[1024];
 	PQExpBuffer sql = NULL;
 	PGresult* result = NULL;
 
@@ -562,8 +658,8 @@ void update_segment_failed_tmpdir
 	conn = PQconnectdb(conninfo);
 	if ((libpqres = PQstatus(conn)) != CONNECTION_OK)
 	{
-		elog(WARNING, "Fail to connect database when update segment's failed tmpdir "
-					  "in segment configuration catalog table, error code: %d, %s",
+		elog(WARNING, "Fail to connect database when add a new row into "
+					  "segment configuration history catalog table, error code: %d, %s",
 					  libpqres,
 					  PQerrorMessage(conn));
 		PQfinish(conn);
@@ -573,24 +669,28 @@ void update_segment_failed_tmpdir
 	result = PQexec(conn, "BEGIN");
 	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
 	{
-		elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
-					  "in segment configuration catalog table, reason : %s",
+		elog(WARNING, "Fail to run SQL: %s when add a new row into "
+					  "segment configuration history catalog table, reason : %s",
 					  "BEGIN",
 					  PQresultErrorMessage(result));
 		goto cleanup;
 	}
 	PQclear(result);
 
+	TimestampTz curtime = GetCurrentTimestamp();
+	const char *curtimestr = timestamptz_to_str(curtime);
 	sql = createPQExpBuffer();
-	appendPQExpBuffer(sql, "UPDATE gp_segment_configuration SET "
-						   "status='%c', failed_tmpdir_num = '%d', failed_tmpdir = '%s' "
-						   "WHERE registration_order=%d",
-						   status, failedNum, failedTmpDir, id);
+	appendPQExpBuffer(sql,
+					  "INSERT INTO gp_configuration_history"
+					  "(time, registration_order, hostname, description) "
+					  "VALUES ('%s','%d','%s','%s')",
+					  curtimestr, id, hostname, description);
+
 	result = PQexec(conn, sql->data);
 	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
 	{
-		elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
-					  "in segment configuration catalog table, reason : %s",
+		elog(WARNING, "Fail to run SQL: %s when add a new row into "
+				      "segment configuration history catalog table, reason : %s",
 					  sql->data,
 					  PQresultErrorMessage(result));
 		goto cleanup;
@@ -600,18 +700,16 @@ void update_segment_failed_tmpdir
 	result = PQexec(conn, "COMMIT");
 	if (!result || PQresultStatus(result) != PGRES_COMMAND_OK)
 	{
-		elog(WARNING, "Fail to run SQL: %s when update segment's failed tmpdir "
-					  "in segment configuration catalog table, reason : %s",
+		elog(WARNING, "Fail to run SQL: %s when add a new row into "
+					  "segment configuration history catalog table, reason : %s",
 					  "COMMIT",
 					  PQresultErrorMessage(result));
 		goto cleanup;
 	}
 
-	elog(LOG, "Update a segment's failed tmpdir:"
-			  "status to '%c', failed_tmpdir_num to '%d', failed_tmpdir to '%s' "
-			  "in segment configuration catalog table,"
-			  "registration_order : %d",
-			  status, failedNum, failedTmpDir, id);
+	elog(LOG, "Add a new row into segment configuration history catalog table,"
+			  "time: %s, registration order:%d, hostname:%s, description:%s",
+			  curtimestr, id, hostname, description);
 
 cleanup:
 	if(sql)
@@ -629,8 +727,7 @@ cleanup:
  * port : port of this segment
  * role : role of this segment
  * status : up or down
- * failed_tmpdir_num : the number of failed temporary directory
- * failed_tmpdir : failed temporary directory, separated by comma
+ * description : the description of this segment's current status
  */
 void add_segment_config_row(int32_t id,
 							char* hostname,
@@ -638,9 +735,7 @@ void add_segment_config_row(int32_t id,
 							uint32_t port,
 							char role,
 							char status,
-							uint32_t
-							failed_tmpdir_num,
-							char* failed_tmpdir)
+							char* description)
 {
 	int	libpqres = CONNECTION_OK;
 	PGconn *conn = NULL;
@@ -677,10 +772,10 @@ void add_segment_config_row(int32_t id,
 	{
 		appendPQExpBuffer(sql,
 						  "INSERT INTO gp_segment_configuration"
-						  "(registration_order,role,status,port,hostname,address,failed_tmpdir_num,failed_tmpdir) "
+						  "(registration_order,role,status,port,hostname,address,description) "
 						  "VALUES "
-						  "(%d,'%c','%c',%d,'%s','%s',%d,'%s')",
-						  id,role,status,port,hostname,address,failed_tmpdir_num,failed_tmpdir);
+						  "(%d,'%c','%c',%d,'%s','%s','%s')",
+						  id,role,status,port,hostname,address,description);
 	}
 	else
 	{
@@ -713,8 +808,8 @@ void add_segment_config_row(int32_t id,
 
 	elog(LOG, "Add a new row into segment configuration catalog table,"
 			  "registration order:%d, role:%c, status:%c, port:%d, "
-			  "hostname:%s, address:%s, failed_tmpdir_num:%d, failed_tmpdir:%s",
-			  id, role, status, port, hostname, address, failed_tmpdir_num, failed_tmpdir);
+			  "hostname:%s, address:%s, description:%s",
+			  id, role, status, port, hostname, address, description);
 
 cleanup:
 	if(sql)
@@ -758,41 +853,38 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 	 */
 	hostname 	= GET_SEGINFO_HOSTNAME(&(segstat->Info));
 	hostnamelen = segstat->Info.HostNameLen;
-    res = getSegIDByHostName(hostname, hostnamelen, &segid);
-    if ( res != FUNC_RETURN_OK )
-    {
-    	/* Try addresses of this machine. */
-    	for ( int i = 0 ; i < segstat->Info.HostAddrCount ; ++i )
-    	{
-
-    		elog(DEBUG5, "Resource manager checks host ip (%d)th to get segment.", i);
-    		AddressString addr = NULL;
-    		getSegInfoHostAddrStr(&(segstat->Info), i, &addr);
-    		if ( strcmp(addr->Address, IPV4_DOT_ADDR_LO) == 0 &&
-    			 segstat->Info.HostAddrCount > 1 )
-    		{
-    			/*
-    			 * If the host has only one address as 127.0.0.1, we have to
-    			 * save it to track the only one segment, otherwise, we skip
-    			 * 127.0.0.1 address in resource pool. Because, each node has
-    			 * one entry of this address.
-    			 */
-    			continue;
-    		}
+	res = getSegIDByHostName(hostname, hostnamelen, &segid);
+	if ( res != FUNC_RETURN_OK )
+	{
+		/* Try addresses of this machine. */
+		for ( int i = 0 ; i < segstat->Info.HostAddrCount ; ++i )
+		{
+			elog(DEBUG5, "Resource manager checks host ip (%d)th to get segment.", i);
+			AddressString addr = NULL;
+			getSegInfoHostAddrStr(&(segstat->Info), i, &addr);
+			if ( strcmp(addr->Address, IPV4_DOT_ADDR_LO) == 0 &&
+				 segstat->Info.HostAddrCount > 1 )
+			{
+				/*
+				 * If the host has only one address as 127.0.0.1, we have to
+				 * save it to track the only one segment, otherwise, we skip
+				 * 127.0.0.1 address in resource pool. Because, each node has
+				 * one entry of this address.
+				 */
+				continue;
+			}
 
-    		res = getSegIDByHostAddr((uint8_t *)(addr->Address), addr->Length, &segid);
-    		if ( res == FUNC_RETURN_OK )
-    		{
-    			break;
-    		}
-    	}
-    }
+			res = getSegIDByHostAddr((uint8_t *)(addr->Address), addr->Length, &segid);
+			if ( res == FUNC_RETURN_OK )
+			{
+				break;
+			}
+		}
+	}
 
-    /* CASE 1. It is a new host. */
+	/* CASE 1. It is a new host. */
 	if ( res != FUNC_RETURN_OK )
 	{
-		uint8_t reportStatus = segstat->FTSAvailable;
-
 		/* Create machine information and corresponding resource information. */
 		segresource = createSegResource(segstat);
 
@@ -815,7 +907,7 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 		setSimpleStringRef(&hostnamekey, hostname, hostnamelen);
 
 		oldval = setHASHTABLENode(&(PRESPOOL->SegmentHostNameIndexed),
-						  	  	  TYPCONVERT(void *, &hostnamekey),
+								  TYPCONVERT(void *, &hostnamekey),
 								  TYPCONVERT(void *, segid),
 								  false /* There should be no old value. */);
 		if ( oldval != NULL )
@@ -844,22 +936,39 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 					  	  	       TYPCONVERT(void *, &hostaddrkey)) == NULL )
 			{
 				setHASHTABLENode( &(PRESPOOL->SegmentHostAddrIndexed),
-							  	  TYPCONVERT(void *, &hostaddrkey),
+								  TYPCONVERT(void *, &hostaddrkey),
 								  TYPCONVERT(void *, segid),
 								  false /* There should be no old value. */);
 
 				elog(LOG, "Resource manager tracked ip address '%.*s' for host '%s'",
 						  hostaddrkey.Len, hostaddrkey.Array,
-					      hostname);
+						  hostname);
 			}
 		}
 
 		/*
-		 * This is a new host registration. Normally the status is available,
-		 * But if the number of failed temporary directory exceeds guc,
+		 * If in GRM mode, the new registered segment is marked
+		 * as DOWN, since master hasn't get a cluster report for it
+		 * from global Resource Manager.
+		 */
+		if (DRMGlobalInstance->ImpType != NONE_HAWQ2)
+		{
+			segresource->Stat->StatusDesc |= SEG_STATUS_NO_GRM_NODE_REPORT;
+		}
+
+		/*
+		 * If there is any failed temporary directory,
 		 * this segment is considered as unavailable.
 		 */
-		setSegResHAWQAvailability(segresource, reportStatus);
+		if (segresource->Stat->FailedTmpDirNum != 0)
+		{
+			segresource->Stat->StatusDesc |= SEG_STATUS_FAILED_TMPDIR;
+		}
+
+		if (segresource->Stat->StatusDesc == 0)
+		{
+			setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_AVAILABLE);
+		}
 
 		/* Add this node into the table gp_segment_configuration */
 		AddressString straddr = NULL;
@@ -868,16 +977,26 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 
 		if (Gp_role != GP_ROLE_UTILITY)
 		{
+			SimpStringPtr description = build_segment_status_description(segresource->Stat);
 			add_segment_config_row (segid+REGISTRATION_ORDER_OFFSET,
 									hostname,
 									straddr->Address,
 									segresource->Stat->Info.port,
 									SEGMENT_ROLE_PRIMARY,
-									segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
+									IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
 										SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
-									segresource->Stat->FailedTmpDirNum,
-									segresource->Stat->FailedTmpDirNum == 0 ?
-										"":GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
+									(description->Len > 0)?description->Str:"");
+
+			add_segment_history_row(segid+REGISTRATION_ORDER_OFFSET,
+									hostname,
+									IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+										SEG_STATUS_DESCRIPTION_UP:description->Str);
+
+			if (description != NULL)
+			{
+				freeSimpleStringContent(description);
+				rm_pfree(PCONTEXT, description);
+			}
 		}
 
 		if (segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE)
@@ -901,37 +1020,78 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 	else {
 		segresource = getSegResource(segid);
 		Assert(segresource != NULL);
+		uint32_t oldStatusDesc = segresource->Stat->StatusDesc;
 		uint8_t oldStatus = segresource->Stat->FTSAvailable;
-		bool statusChanged = oldStatus != segstat->FTSAvailable;
 
 		/*
-		 * Check if RM process is restarted in this segment.
 		 * If the latest reported RM process startup timestamp doesn't
 		 * match the previous, master RM consider segment's RM process
 		 * has restarted.
 		 * In rare case, the system's time is reset and segment's RM process
 		 * happen to get a same timestamp with previous one.
 		 */
-		if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp)
+		if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp &&
+				(segresource->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) == 0 &&
+				(segresource->Stat->StatusDesc & SEG_STATUS_COMMUNICATION_ERROR) == 0 &&
+				(segresource->Stat->StatusDesc & SEG_STATUS_FAILED_PROBING_SEGMENT) == 0)
 		{
 			/*
-			 * This segment's RM process has restarted,
-			 * we should clean up old status, so mark it down.
+			 * This segment's RM process has restarted.
+			 * if StatusDesc doesn't have heartbeat timeout flag, or communication error,
+			 * or failed probing segment flag, this segment is set to DOWN.
+			 * It will be set to UP when reports a new heartbeat.
 			 */
-			if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE && !statusChanged)
-			{
-				segstat->FTSAvailable = RESOURCE_SEG_STATUS_UNAVAILABLE;
-				statusChanged = true;
-			}
+			segresource->Stat->StatusDesc |= SEG_STATUS_RM_RESET;
 			segresource->Stat->RMStartTimestamp = segstat->RMStartTimestamp;
 			elog(LOG, "Master RM finds segment:%s 's RM process has restarted. "
-					  "old status:%d, new status:%d",
+					  "old status:%d",
 					  GET_SEGRESOURCE_HOSTNAME(segresource),
-					  oldStatus,
-					  segstat->FTSAvailable);
+					  oldStatus);
+		}
+		else
+		{
+			if (segresource->Stat->RMStartTimestamp != segstat->RMStartTimestamp)
+			{
+				segresource->Stat->RMStartTimestamp = segstat->RMStartTimestamp;
+			}
+			/*
+			 * Now clear heartbeat timeout flag, RM reset flag,
+			 * no response flag, communication error flag
+			 * and RM reset flag in StatusDesc
+			 */
+			if ((segresource->Stat->StatusDesc & SEG_STATUS_HEARTBEAT_TIMEOUT) != 0)
+			{
+				segresource->Stat->StatusDesc &= ~SEG_STATUS_HEARTBEAT_TIMEOUT;
+				elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+							"clear its heartbeat timeout flag",
+							GET_SEGRESOURCE_HOSTNAME(segresource));
+			}
+			if ((segresource->Stat->StatusDesc & SEG_STATUS_FAILED_PROBING_SEGMENT) != 0)
+			{
+				segresource->Stat->StatusDesc &= ~SEG_STATUS_FAILED_PROBING_SEGMENT;
+				elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+							"clear its failed probing segment flag",
+							GET_SEGRESOURCE_HOSTNAME(segresource));
+			}
+			if ((segresource->Stat->StatusDesc & SEG_STATUS_COMMUNICATION_ERROR) != 0)
+			{
+				segresource->Stat->StatusDesc &= ~SEG_STATUS_COMMUNICATION_ERROR;
+				elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+							"clear its communication error flag",
+							GET_SEGRESOURCE_HOSTNAME(segresource));
+			}
+			if ((segresource->Stat->StatusDesc & SEG_STATUS_RM_RESET) != 0)
+			{
+				segresource->Stat->StatusDesc &= ~SEG_STATUS_RM_RESET;
+				elog(RMLOG, "Master RM gets heartbeat report from segment:%s, "
+							"clear its RM reset flag",
+							GET_SEGRESOURCE_HOSTNAME(segresource));
+			}
 		}
 
-		/* Check if temporary directory path is changed */
+		/*
+		 * If temporary directory path is changed, update SegStatData
+		 */
 		bool tmpDirChanged = false;
 		if (segresource->Stat->FailedTmpDirNum != segstat->FailedTmpDirNum)
 		{
@@ -944,160 +1104,162 @@ int addHAWQSegWithSegStat(SegStat segstat, bool *capstatchanged)
 						GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)) != 0)
 			{
 				tmpDirChanged = true;
-				elog(LOG, "Resource manager finds segment %s(%d) 's "
-						  "failed temporary directory is changed from "
-						  "'%s' to '%s'",
-						  GET_SEGRESOURCE_HOSTNAME(segresource),
-						  segid,
-						  GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
-						  GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
 			}
 		}
 
-		/*
-		 * Either the FTSAvailable or the failed temporary directory
-		 * of this segment is changed.
-		 */
-		if (statusChanged || tmpDirChanged)
+		if (tmpDirChanged)
 		{
-			if (statusChanged && !tmpDirChanged)
-			{
-				if (Gp_role != GP_ROLE_UTILITY)
-				{
-					update_segment_status(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
-											segstat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
-											SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN);
-				}
-
-				setSegResHAWQAvailability(segresource, segstat->FTSAvailable);
-				/*
-				 * Segment is set from up to down, return resource.
-				 */
-				if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
-				{
-					*capstatchanged = true;
-					returnAllGRMResourceFromSegment(segresource);
-				}
+			elog(RMLOG, "Resource manager is going to set segment %s(%d) 's "
+						"failed temporary directory from "
+						"'%s' to '%s'",
+						GET_SEGRESOURCE_HOSTNAME(segresource),
+						segid,
+						segresource->Stat->FailedTmpDirNum == 0 ?
+							"" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
+						segstat->FailedTmpDirNum == 0 ?
+							"" : GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
+
+			int old = segresource->Stat->Info.FailedTmpDirLen == 0 ?
+									0 :__SIZE_ALIGN64(segresource->Stat->Info.FailedTmpDirLen+1);
+			int new = segstat->Info.FailedTmpDirLen == 0 ?
+									0 : __SIZE_ALIGN64(segstat->Info.FailedTmpDirLen+1);
+
+			int current = segresource->Stat->Info.Size -
+					(segresource->Stat->Info.HostNameOffset + __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1));
+			if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
+				current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
+			if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
+				current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
 
-				elog(LOG, "Master resource manager sets segment %s(%d)'s status "
-						  "to %c",
-						  GET_SEGRESOURCE_HOSTNAME(segresource),
-						  segid,
-						  segstat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
-									SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN);
+			/*
+			 * repalloc memory if new size exceeds the old one.
+			 * we don't shrink memory size if new size is less than the old one.
+			 */
+			if (new > old && current < new)
+			{
+				SegStat newSegStat = rm_repalloc(PCONTEXT,
+												 segresource->Stat,
+												 offsetof(SegStatData, Info) +
+												 segresource->Stat->Info.Size + (new - old));
+				segresource->Stat = newSegStat;
+				segresource->Stat->Info.Size += (new - old);
 			}
-			else
+
+			if (segresource->Stat->Info.FailedTmpDirOffset == 0)
 			{
-				/*
-				*  Failed temporary directory is changed,
-				*  if the length of new failed temporary directory exceeds the old one,
-				*  we need to repalloc SegInfoData
-				*/
-				elog(RMLOG, "Master resource manager is going to set segment %s(%d)'s "
-							"failed temporary directory from '%s' to '%s'",
-							GET_SEGRESOURCE_HOSTNAME(segresource),
-							segid,
-							segresource->Stat->FailedTmpDirNum == 0 ?
-								"" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
-							segstat->FailedTmpDirNum == 0 ?
-								"" : GET_SEGINFO_FAILEDTMPDIR(&segstat->Info));
-
-				int old = segresource->Stat->Info.FailedTmpDirLen == 0 ?
-										0 :__SIZE_ALIGN64(segresource->Stat->Info.FailedTmpDirLen+1);
-				int new = segstat->Info.FailedTmpDirLen == 0 ?
-										0 : __SIZE_ALIGN64(segstat->Info.FailedTmpDirLen+1);
-
-				int current = segresource->Stat->Info.Size -
-						(segresource->Stat->Info.HostNameOffset + __SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1));
+				Assert(segresource->Stat->FailedTmpDirNum == 0);
+				segresource->Stat->Info.FailedTmpDirOffset = segresource->Stat->Info.HostNameOffset +
+																__SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1);
 				if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
-					current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
+					segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
 				if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
-					current -= __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
+					segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
+			}
 
-				/*
-				 * repalloc memory if new size exceeds the old one.
-				 * we don't shrink memory size if new size is less than the old one.
-				 */
-				if (new > old && current < new)
-				{
-					SegStat newSegStat = rm_repalloc(PCONTEXT,
-													 segresource->Stat,
-													 offsetof(SegStatData, Info) +
-													 segresource->Stat->Info.Size + (new - old));
-					segresource->Stat = newSegStat;
-					segresource->Stat->Info.Size += (new - old);
-				}
+			/* Clear old failed temporary directory string in SegInfoData */
+			memset((char *)&segresource->Stat->Info +
+					segresource->Stat->Info.FailedTmpDirOffset,
+					0,
+					segresource->Stat->Info.Size -
+					segresource->Stat->Info.FailedTmpDirOffset);
 
-				if (segresource->Stat->Info.FailedTmpDirOffset == 0)
-				{
-					Assert(segresource->Stat->FailedTmpDirNum == 0);
-					segresource->Stat->Info.FailedTmpDirOffset = segresource->Stat->Info.HostNameOffset +
-																	__SIZE_ALIGN64(segresource->Stat->Info.HostNameLen+1);
-					if (segresource->Stat->Info.GRMHostNameLen != 0 && segresource->Stat->Info.GRMHostNameOffset != 0)
-						segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMHostNameLen+1);
-					if (segresource->Stat->Info.GRMRackNameLen != 0 && segresource->Stat->Info.GRMRackNameOffset != 0)
-						segresource->Stat->Info.FailedTmpDirOffset += __SIZE_ALIGN64(segresource->Stat->Info.GRMRackNameLen+1);
-				}
+			if (segstat->FailedTmpDirNum != 0)
+			{
+				/* Update failed temporary directory string in SegInfoData */
+				memcpy((char *)&segresource->Stat->Info + segresource->Stat->Info.FailedTmpDirOffset,
+						GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
+						strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
+			}
+			else
+			{
+				segresource->Stat->Info.FailedTmpDirOffset = 0;
+			}
+			segresource->Stat->Info.FailedTmpDirLen = segstat->Info.FailedTmpDirLen;
+			segresource->Stat->FailedTmpDirNum = segstat->FailedTmpDirNum;
 
-				/* clear old failed temporary directory string in SegInfoData */
-				memset((char *)&segresource->Stat->Info +
-						segresource->Stat->Info.FailedTmpDirOffset,
-						0,
-						segresource->Stat->Info.Size -
-						segresource->Stat->Info.FailedTmpDirOffset);
+			elog(LOG, "Resource manager change segment %s(%d) 's "
+					  "failed temporary directory is changed to '%s'",
+					  GET_SEGRESOURCE_HOSTNAME(segresource),
+					  segid,
+					  segresource->Stat->FailedTmpDirNum == 0 ?
+						"" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
+
+			elog(RMLOG, "After resource manager "
+						"updates segment failed temporary directory, "
+						"GRM hostname:%s, GRM rackname:%s",
+						segresource->Stat->Info.GRMHostNameLen == 0 ?
+							"":GET_SEGINFO_GRMHOSTNAME(&(segresource->Stat->Info)),
+						segresource->Stat->Info.GRMRackNameLen == 0 ?
+							"":GET_SEGINFO_GRMRACKNAME(&(segresource->Stat->Info)));
+		}
 
-				if (segstat->FailedTmpDirNum != 0)
-				{
-					memcpy((char *)&segresource->Stat->Info + segresource->Stat->Info.FailedTmpDirOffset,
-							GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
-							strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
-				}
-				else
-				{
-					segresource->Stat->Info.FailedTmpDirOffset = 0;
-				}
-				segresource->Stat->Info.FailedTmpDirLen = segstat->Info.FailedTmpDirLen;
-				segresource->Stat->FailedTmpDirNum = segstat->FailedTmpDirNum;
-
-				elog(RMLOG, "After resource manager "
-							"updates segment failed temporary directory, "
-							"GRM hostname:%s, GRM rackname:%s",
-							segresource->Stat->Info.GRMHostNameLen == 0 ?
-								"":GET_SEGINFO_GRMHOSTNAME(&(segresource->Stat->Info)),
-							segresource->Stat->Info.GRMRackNameLen == 0 ?
-								"":GET_SEGINFO_GRMRACKNAME(&(segresource->Stat->Info)));
-
-				setSegResHAWQAvailability(segresource, segstat->FTSAvailable);
-				if (Gp_role != GP_ROLE_UTILITY)
-				{
-					update_segment_failed_tmpdir(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
-												 segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
-																			SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
-												 segresource->Stat->FailedTmpDirNum,
-												 segresource->Stat->FailedTmpDirNum == 0 ?
-															 "" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info));
-				}
+		/* Set or clear failed temporary directory flag */
+		if (segresource->Stat->FailedTmpDirNum != 0)
+		{
+			segresource->Stat->StatusDesc |= SEG_STATUS_FAILED_TMPDIR;
+		}
+		else
+		{
+			if ((segresource->Stat->StatusDesc & SEG_STATUS_FAILED_TMPDIR) != 0)
+				segresource->Stat->StatusDesc &= ~SEG_STATUS_FAILED_TMPDIR;
+		}
+
+		if (segresource->Stat->StatusDesc == 0)
+		{
+			if (oldStatus == RESOURCE_SEG_STATUS_UNAVAILABLE ||
+					oldStatus == RESOURCE_SEG_STATUS_UNSET)
+				setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_AVAILABLE);
+		}
+		else
+		{
+			if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+				setSegResHAWQAvailability(segresource, RESOURCE_SEG_STATUS_UNAVAILABLE);
+		}
 
-				if (statusChanged)
+		if (oldStatusDesc != segresource->Stat->StatusDesc ||
+				tmpDirChanged)
+		{
+			/*
+			 * StatusDesc or failed temporary directory path is changed,
+			 * update the gp_segment_configuration table and insert a row
+			 * into gp_configuration_history table
+			 */
+			if (Gp_role != GP_ROLE_UTILITY)
+			{
+				SimpStringPtr description = build_segment_status_description(segresource->Stat);
+				update_segment_status(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
+										IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+																SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+										(description->Len > 0)?description->Str:"");
+
+				elog(LOG, "Resource manager update node(%s) information with heartbeat report,"
+							"status:'%c', description:%s",
+							GET_SEGRESOURCE_HOSTNAME(segresource),
+							IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+								SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+							(description->Len > 0)?description->Str:"");
+
+				add_segment_history_row(segresource->Stat->ID + REGISTRATION_ORDER_OFFSET,
+										GET_SEGRESOURCE_HOSTNAME(segresource),
+										IS_SEGSTAT_FTSAVAILABLE(segresource->Stat) ?
+											SEG_STATUS_DESCRIPTION_UP:description->Str);
+				if (description != NULL)
 				{
-					*capstatchanged = true;
-					/*
-					 * Segment is set from up to down, return resource.
-					 */
-					if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
-					{
-						returnAllGRMResourceFromSegment(segresource);
-					}
+					freeSimpleStringContent(description);
+					rm_pfree(PCONTEXT, description);
 				}
+			}
+		}
 
-				elog(LOG, "Master resource manager sets segment %s(%d)'s "
-							"failed temporary directory to '%s', status:%c",
-							GET_SEGRESOURCE_HOSTNAME(segresource),
-							segid,
-							segresource->Stat->FailedTmpDirNum == 0 ?
-								"" : GET_SEGINFO_FAILEDTMPDIR(&segresource->Stat->Info),
-							segresource->Stat->FTSAvailable == RESOURCE_SEG_STATUS_AVAILABLE ?
-								SEGMENT_STATUS_UP : SEGMENT_STATUS_DOWN);
+		if (oldStatus != segresource->Stat->FTSAvailable)
+		{
+			*capstatchanged = true;
+			/*
+			 * Segment is set from up to down, return resource.
+			 */
+			if (oldStatus == RESOURCE_SEG_STATUS_AVAILABLE)
+			{
+				returnAllGRMResourceFromSegment(segresource);
 			}
 		}
 
@@ -1161,6 +1323,7 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 	SegResource	 	 segres	 		 = NULL;
 	SegStat	 	 	 newSegStat	     = NULL;
 	int32_t		 	 segid			 = SEGSTAT_ID_INVALID;
+	bool 			statusDescChange = false;
 
 	/* Anyway, the host GRM capacity is updated here if the cluster level
 	 * capacity is fixed.
@@ -1305,10 +1468,8 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 	elog(RMLOG, "After resource manager "
 				"updates segment info's GRM host name and rack name, "
 				"failed temporary directory: %s",
-				segres->Stat->FailedTmpDirNum == 0 ? "":GET_SEGINFO_FAILEDTMPDIR(&(segres->Stat->Info)));
-
-	/* Always set segment global resource manager available. */
-	setSegResGLOBAvailability(segres, RESOURCE_SEG_STATUS_AVAILABLE);
+				segres->Stat->FailedTmpDirNum == 0 ?
+					"":GET_SEGINFO_FAILEDTMPDIR(&(segres->Stat->Info)));
 
 	if ( segres->Stat->GRMTotalMemoryMB != segstat->GRMTotalMemoryMB ||
 		 segres->Stat->GRMTotalCore     != segstat->GRMTotalCore )
@@ -1320,14 +1481,20 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 		segres->Stat->GRMTotalMemoryMB = segstat->GRMTotalMemoryMB;
 		segres->Stat->GRMTotalCore     = segstat->GRMTotalCore;
 
-		/* Update overall cluster resource capacity. */
-		minusResourceBundleData(&(PRESPOOL->GRMTotal), oldgrmmem, oldgrmcore);
-		addResourceBundleData(&(PRESPOOL->GRMTotal),
-						  	  segres->Stat->GRMTotalMemoryMB,
-							  segres->Stat->GRMTotalCore);
+		/*
+		 * If this segment is FTS available, then
+		 * update overall cluster resource capacity.
+		 */
+		if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
+		{
+			minusResourceBundleData(&(PRESPOOL->GRMTotal), oldgrmmem, oldgrmcore);
+			addResourceBundleData(&(PRESPOOL->GRMTotal),
+								  segres->Stat->GRMTotalMemoryMB,
+								  segres->Stat->GRMTotalCore);
+		}
 
 		elog(LOG, "Resource manager finds host %s capacity changed from "
-				    "GRM (%d MB, %d CORE) to GRM (%d MB, %d CORE)",
+					"GRM (%d MB, %d CORE) to GRM (%d MB, %d CORE)",
 					GET_SEGRESOURCE_HOSTNAME(segres),
 					oldgrmmem,
 					oldgrmcore,
@@ -1335,6 +1502,49 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 					segres->Stat->GRMTotalCore);
 	}
 
+	segres->Stat->GRMHandled = true;
+	/* Clear no GRM node report flag. */
+	if ((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT) != 0)
+	{
+		segres->Stat->StatusDesc &= ~SEG_STATUS_NO_GRM_NODE_REPORT;
+		statusDescChange = true;
+	}
+
+	/*
+	 * If get GRM node report, and there is no other flag,
+	 * mark this segment to UP
+	 */
+	if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) && segres->Stat->StatusDesc == 0)
+	{
+		Assert(statusDescChange == true);
+		setSegResHAWQAvailability(segres, RESOURCE_SEG_STATUS_AVAILABLE);
+	}
+
+	if (statusDescChange && Gp_role != GP_ROLE_UTILITY)
+	{
+		SimpStringPtr description = build_segment_status_description(segres->Stat);
+		update_segment_status(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+								IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+									SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+								 (description->Len > 0)?description->Str:"");
+		add_segment_history_row(segres->Stat->ID + REGISTRATION_ORDER_OFFSET,
+								GET_SEGRESOURCE_HOSTNAME(segres),
+								IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+									SEG_STATUS_DESCRIPTION_UP:description->Str);
+
+		elog(LOG, "Resource manager update node(%s) information with yarn node report,"
+					"status:'%c', description:%s",
+					GET_SEGRESOURCE_HOSTNAME(segres),
+					IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ?
+						SEGMENT_STATUS_UP:SEGMENT_STATUS_DOWN,
+					(description->Len > 0)?description->Str:"");
+		if (description != NULL)
+		{
+			freeSimpleStringContent(description);
+			rm_pfree(PCONTEXT, description);
+		}
+	}
+
 	int32_t curratio = 0;
 	if (DRMGlobalInstance->ImpType == YARN_LIBYARN &&
 		segres->Stat->GRMTotalMemoryMB > 0 &&
@@ -1347,7 +1557,7 @@ int updateHAWQSegWithGRMSegStat( SegStat segstat)
 	return FUNC_RETURN_OK;
 }
 
-void setAllSegResourceGRMUnavailable(void)
+void setAllSegResourceGRMUnhandled(void)
 {
 	List 	 *allsegres = NULL;
 	ListCell *cell		= NULL;
@@ -1356,7 +1566,7 @@ void setAllSegResourceGRMUnavailable(void)
 	foreach(cell, allsegres)
 	{
 		SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
-		setSegResGLOBAvailability(segres, RESOURCE_SEG_STATUS_UNAVAILABLE);
+		segres->Stat->GRMHandled = false;
 	}
 	freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
 }
@@ -1432,7 +1642,6 @@ SegResource createSegResource(SegStat segstat)
 	res->LastUpdateTime  = gettime_microsec();
 	res->RUAlivePending  = false;
 	res->Stat->FTSAvailable = RESOURCE_SEG_STATUS_UNSET;
-	res->Stat->GRMAvailable = RESOURCE_SEG_STATUS_UNSET;
 
 	for ( int i = 0 ; i < RESOURCE_QUEUE_RATIO_SIZE ; ++i )
 	{
@@ -1455,13 +1664,6 @@ int setSegStatHAWQAvailability( SegStat segstat, uint8_t newstatus)
 	return res;
 }
 
-int setSegStatGLOBAvailability( SegStat segstat, uint8_t newstatus)
-{
-	int res = segstat->GRMAvailable;
-	segstat->GRMAvailable = newstatus;
-	return res;
-}
-
 /* Set hawq status of host, return the old status */
 int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
 {
@@ -1499,12 +1701,7 @@ int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
 							  segres->Stat->GRMTotalCore);
 
 		addNewResourceToResourceManagerByBundle(&(segres->Allocated));
-		if ( (DRMGlobalInstance->ImpType == NONE_HAWQ2) ||
-			 (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
-			  IS_SEGSTAT_GRMAVAILABLE(segres->Stat)))
-		{
-			PRESPOOL->AvailNodeCount++;
-		}
+		PRESPOOL->AvailNodeCount++;
 	}
 	else
 	{
@@ -1528,12 +1725,6 @@ int setSegResHAWQAvailability( SegResource segres, uint8_t newstatus)
 	return res;
 }
 
-int setSegResGLOBAvailability( SegResource segres, uint8_t newstatus)
-{
-	int res = setSegStatGLOBAvailability(segres->Stat, newstatus);
-	return res;
-}
-
 /* Generate HAWQ host report. */
 void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
 {
@@ -1549,15 +1740,14 @@ void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
 		static char reporthead[256];
 
 		int headsize = sprintf(reporthead,
-                			   "SEGMENT:ID=%d, "
-                			   "HAWQAVAIL=%d,GLOBAVAIL=%d. "
-                			   "FTS( %d MB, %d CORE). "
+							   "SEGMENT:ID=%d, "
+							   "HAWQAVAIL=%d. "
+							   "FTS( %d MB, %d CORE). "
 							   "GRM( %d MB, %d CORE). "
 							   "MEM=%d(%d) MB. "
 							   "CORE=%lf(%lf).\n",
 							   seg->Stat->ID,
 							   seg->Stat->FTSAvailable,
-							   seg->Stat->GRMAvailable,
 							   seg->Stat->FTSTotalMemoryMB,
 							   seg->Stat->FTSTotalCore,
 							   seg->Stat->GRMTotalMemoryMB,
@@ -1565,7 +1755,7 @@ void generateSegResourceReport(int32_t segid, SelfMaintainBuffer buff)
 							   seg->Allocated.MemoryMB,
 							   seg->Available.MemoryMB,
 							   seg->Allocated.Core,
-                			   seg->Available.Core);
+							   seg->Available.Core);
 		appendSelfMaintainBuffer(buff, reporthead, headsize);
 		generateSegInfoReport(&(seg->Stat->Info), buff);
 	}
@@ -1655,13 +1845,12 @@ void  generateSegStatReport(SegStat segstat, SelfMaintainBuffer buff)
 	static char reporthead[256];
 	int reportheadlen = 0;
 	reportheadlen = sprintf(reporthead,
-            "NODE:ID=%d,HAWQ %s, GRM %s, "
-            "HAWQ CAP (%d MB, %lf CORE), "
+			"NODE:ID=%d,HAWQ %s, "
+			"HAWQ CAP (%d MB, %lf CORE), "
 			"GRM CAP(%d MB, %lf CORE),",
-            segstat->ID,
+			segstat->ID,
 			segstat->FTSAvailable ? "AVAIL" : "UNAVAIL",
-			segstat->GRMAvailable ? "AVAIL" : "UNAVAIL",
-            segstat->FTSTotalMemoryMB,
+			segstat->FTSTotalMemoryMB,
 			segstat->FTSTotalCore * 1.0,
 			segstat->GRMTotalMemoryMB,
 			segstat->GRMTotalCore * 1.0);
@@ -3004,9 +3193,9 @@ void returnAllGRMResourceFromSegment(SegResource segres)
 
 /*
  * Go through each segment, and return the GRM containers in those segments
- * global resource manager unavailable.
+ * with GRM unavailable or FTS unavailable.
  */
-void returnAllGRMResourceFromGRMUnavailableSegments(void)
+void returnAllGRMResourceFromUnavailableSegments(void)
 {
 	List 	 *allsegres = NULL;
 	ListCell *cell		= NULL;
@@ -3016,7 +3205,7 @@ void returnAllGRMResourceFromGRMUnavailableSegments(void)
 	foreach(cell, allsegres)
 	{
 		SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
-		if ( IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+		if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
 		{
 			continue;
 		}
@@ -4208,8 +4397,7 @@ void refreshAvailableNodeCount(void)
 		SegResource segres = (SegResource)(pair->Value);
 		Assert( segres != NULL );
 
-		if ( IS_SEGSTAT_FTSAVAILABLE(segres->Stat) &&
-			 IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+		if ( IS_SEGSTAT_FTSAVAILABLE(segres->Stat) )
 		{
 			PRESPOOL->AvailNodeCount++;
 		}
@@ -4335,10 +4523,12 @@ void fixClusterMemoryCoreRatio(void)
 		}
 		else
 		{
-			if ( !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
-			{
-				continue;
-			}
+			/*
+			 * If this segment is FTS available,
+			 * RM should get GRM report for it.
+			 */
+			Assert((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT)
+					== 0);
 			memorymb = segres->Stat->GRMTotalMemoryMB;
 			core	 = segres->Stat->GRMTotalCore;
 		}
@@ -4389,10 +4579,12 @@ void fixClusterMemoryCoreRatio(void)
 			}
 			else
 			{
-				if ( !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
-				{
-					continue;
-				}
+				/*
+				 * If this segment is FTS available,
+				 * RM should get GRM report for it.
+				 */
+				Assert((segres->Stat->StatusDesc & SEG_STATUS_NO_GRM_NODE_REPORT)
+						== 0);
 				memorymb = segres->Stat->GRMTotalMemoryMB;
 				core	 = segres->Stat->GRMTotalCore;
 			}
@@ -4563,8 +4755,7 @@ void adjustSegmentCapacityForGRM(SegResource segres)
 	adjustMemoryCoreValue(&(segres->Stat->GRMTotalMemoryMB),
 						  &(segres->Stat->GRMTotalCore));
 
-	if ( !IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
-		 !IS_SEGSTAT_GRMAVAILABLE(segres->Stat))
+	if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
 	{
 		return;
 	}
@@ -4628,9 +4819,8 @@ void dumpResourcePoolHosts(const char *filename)
                     segresource->Stat->FTSTotalCore,
                     segresource->Stat->GRMTotalMemoryMB,
                     segresource->Stat->GRMTotalCore);
-            fprintf(fp, "HOST_AVAILABLITY(HAWQAvailable=%s:GLOBAvailable=%s)\n",
-                    segresource->Stat->FTSAvailable == 0 ? "false" : "true",
-                    segresource->Stat->GRMAvailable == 0 ? "false" : "true");
+            fprintf(fp, "HOST_AVAILABLITY(HAWQAvailable=%s)\n",
+                    segresource->Stat->FTSAvailable == 0 ? "false" : "true");
             fprintf(fp, "HOST_RESOURCE(AllocatedMemory=%d:AllocatedCores=%f:"
             						  "AvailableMemory=%d:AvailableCores=%f:"
             						  "IOBytesWorkload="INT64_FORMAT":"
@@ -4686,3 +4876,66 @@ void dumpResourcePoolHosts(const char *filename)
 
     fclose(fp);
 }
+
+static const char* SegStatusDesc[] = {
+	"heartbeat timeout",
+	"failed probing segment",
+	"communication error",
+	"failed temporary directory",
+	"resource manager process was reset",
+	"no global node report"
+};
+
+/*
+ * Build a string of status description for SegStat.
+ * This string contains the reason why this segment is DOWN.
+ * e.g. the failed temporary directories, heartbeat timeout.
+ */
+SimpStringPtr build_segment_status_description(SegStat segstat)
+{
+	SimpStringPtr description = createSimpleString(PCONTEXT);
+	SelfMaintainBufferData buf;
+	initializeSelfMaintainBuffer(&buf, PCONTEXT);
+	uint32_t tmp,cnt;
+
+	if (segstat->StatusDesc == 0)
+		goto _exit;
+
+	/* Count the number of flags */
+	cnt = tmp = segstat->StatusDesc;
+	while (tmp != 0)
+	{
+		tmp = tmp >> 1;
+		cnt -= tmp;
+	}
+
+	for (int idx = 0, tmp = 0; idx < sizeof(SegStatusDesc)/sizeof(char*); idx++)
+	{
+		if ((segstat->StatusDesc & 1<<idx) != 0)
+		{
+			tmp++;
+			appendSelfMaintainBuffer(&buf, SegStatusDesc[idx], strlen(SegStatusDesc[idx]));
+			if (1<<idx == SEG_STATUS_FAILED_TMPDIR)
+			{
+				appendSelfMaintainBuffer(&buf, ":", 1);
+				appendSelfMaintainBuffer(&buf,
+										 GET_SEGINFO_FAILEDTMPDIR(&segstat->Info),
+										 strlen(GET_SEGINFO_FAILEDTMPDIR(&segstat->Info)));
+			}
+			if (tmp != cnt)
+				appendSelfMaintainBuffer(&buf, ";", 1);
+		}
+	}
+
+	if (buf.Size > 0)
+	{
+		static char zeropad = '\0';
+		appendSMBVar(&buf, zeropad);
+		setSimpleStringNoLen(description, buf.Buffer);
+		destroySelfMaintainBuffer(&buf);
+	}
+
+_exit:
+	return description;
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 00de72d..9af0d7c 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -2631,8 +2631,7 @@ void refreshActualMinGRMContainerPerSeg(void)
 		foreach(cell, allsegres)
 		{
 			SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
-			if ( !IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
-				 !IS_SEGSTAT_GRMAVAILABLE(segres->Stat) )
+			if (IS_SEGSTAT_FTSAVAILABLE(segres->Stat))
 			{
 				continue;
 			}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/backend/utils/gp/segadmin.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c
index 6b49926..775cadb 100644
--- a/src/backend/utils/gp/segadmin.c
+++ b/src/backend/utils/gp/segadmin.c
@@ -277,8 +277,7 @@ gp_add_master_standby(PG_FUNCTION_ARGS)
 	values[Anum_gp_segment_configuration_port - 1] = Int32GetDatum(master->port);
 	values[Anum_gp_segment_configuration_hostname - 1] = PG_GETARG_DATUM(0);
 	values[Anum_gp_segment_configuration_address - 1] = PG_GETARG_DATUM(1);
-	nulls[Anum_gp_segment_configuration_failed_tmpdir_num - 1] = true;
-	nulls[Anum_gp_segment_configuration_failed_tmpdir - 1] = true;
+	nulls[Anum_gp_segment_configuration_description - 1] = true;
 
 	tuple = caql_form_tuple(pcqCtx, values, nulls);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/gp_configuration.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/gp_configuration.h b/src/include/catalog/gp_configuration.h
index eaa0893..ee127e6 100644
--- a/src/include/catalog/gp_configuration.h
+++ b/src/include/catalog/gp_configuration.h
@@ -135,9 +135,10 @@ typedef FormData_gp_configuration *Form_gp_configuration;
    CREATE TABLE gp_configuration_history
    with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)
    (
-   time    timestamp with time zone,
-   dbid    smallint,
-   "desc"    text
+   time               timestamp with time zone,
+   registration_order integer,
+   hostname           text,
+   description        text
    );
 
    TIDYCAT_ENDDEF
@@ -146,8 +147,8 @@ typedef FormData_gp_configuration *Form_gp_configuration;
 /* TIDYCAT_BEGIN_CODEGEN 
 
    WARNING: DO NOT MODIFY THE FOLLOWING SECTION: 
-   Generated by tidycat.pl version 21.
-   on Wed Nov 24 14:13:16 2010
+   Generated by tidycat.pl version 34
+   on Fri Feb 26 10:43:15 2016
 */
 
 
@@ -182,9 +183,10 @@ typedef FormData_gp_configuration *Form_gp_configuration;
 
 CATALOG(gp_configuration_history,5006) BKI_SHARED_RELATION BKI_WITHOUT_OIDS
 {
-	timestamptz	time;	
-	int2		dbid;	
-	text		desc;	
+	timestamptz	time;				
+	int4		registration_order;	
+	text		hostname;			
+	text		description;
 } FormData_gp_configuration_history;
 
 #undef timestamptz
@@ -202,10 +204,11 @@ typedef FormData_gp_configuration_history *Form_gp_configuration_history;
  *		compiler constants for gp_configuration_history
  * ----------------
  */
-#define Natts_gp_configuration_history		3
-#define Anum_gp_configuration_history_time	1
-#define Anum_gp_configuration_history_dbid	2
-#define Anum_gp_configuration_history_desc	3
+#define Natts_gp_configuration_history						4
+#define Anum_gp_configuration_history_time					1
+#define Anum_gp_configuration_history_registration_order	2
+#define Anum_gp_configuration_history_hostname				3
+#define Anum_gp_configuration_history_description			4
 
 
 /* TIDYCAT_END_CODEGEN */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/gp_segment_config.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/gp_segment_config.h b/src/include/catalog/gp_segment_config.h
index f9f2def..fa577f1 100644
--- a/src/include/catalog/gp_segment_config.h
+++ b/src/include/catalog/gp_segment_config.h
@@ -53,8 +53,7 @@
    port               integer    ,
    hostname           text       ,
    address            text       ,
-   failed_tmpdir_num  integer    ,
-   failed_tmpdir      text
+   description        text
    );
 
    create unique index on gp_segment_configuration(registration_order) with (indexid=6106, indexname=gp_segment_config_registration_order_index);
@@ -95,8 +94,7 @@ CATALOG(gp_segment_configuration,5036) BKI_SHARED_RELATION BKI_WITHOUT_OIDS
 	int4	port;				
 	text	hostname;			
 	text	address;			
-	int4	failed_tmpdir_num;	
-	text	failed_tmpdir;		
+	text	description;		
 } FormData_gp_segment_configuration;
 
 
@@ -112,15 +110,14 @@ typedef FormData_gp_segment_configuration *Form_gp_segment_configuration;
  *		compiler constants for gp_segment_configuration
  * ----------------
  */
-#define Natts_gp_segment_configuration						8
+#define Natts_gp_segment_configuration						7
 #define Anum_gp_segment_configuration_registration_order	1
 #define Anum_gp_segment_configuration_role					2
 #define Anum_gp_segment_configuration_status				3
 #define Anum_gp_segment_configuration_port					4
 #define Anum_gp_segment_configuration_hostname				5
 #define Anum_gp_segment_configuration_address				6
-#define Anum_gp_segment_configuration_failed_tmpdir_num		7
-#define Anum_gp_segment_configuration_failed_tmpdir			8
+#define Anum_gp_segment_configuration_description			7
 
 
 /* TIDYCAT_END_CODEGEN */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/catalog/pg_proc.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index caf22a9..a6f8970 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -10196,7 +10196,6 @@ DESCR("Convert a cloned master catalog for use as a segment");
 DATA(insert OID = 5053 ( gp_activate_standby  PGNSP PGUID 12 f f f f v 0 16 f "" _null_ _null_ _null_ gp_activate_standby - _null_ n ));
 DESCR("Activate a standby");
 
-
 /* We cheat in the following two functions: they are technically volatile but */
 /* we can only dispatch them if they're immutable :(. */
 /* gp_add_segment_persistent_entries(int2, int2, _text) => bool */ 
@@ -10285,6 +10284,10 @@ DESCR("Remove an entry from gp_relation_node");
 DATA(insert OID = 5074 ( gp_persistent_relation_node_check  PGNSP PGUID 12 f f f t v 0 6990 f "" _null_ _null_ _null_ gp_persistent_relation_node_check - _null_ n ));
 DESCR("physical filesystem information");
 
+/* gp_remove_segment_history() => bool */
+DATA(insert OID = 5075 ( gp_remove_segment_history  PGNSP PGUID 12 f f f f v 0 16 f "" _null_ _null_ _null_ gp_remove_segment_history - _null_ n ));
+DESCR("Remove all entries from the gp_configuration_history");
+
 /* cosh(float8) => float8 */ 
 DATA(insert OID = 3539 ( cosh  PGNSP PGUID 12 f f f f i 1 701 f "701" _null_ _null_ _null_ dcosh - _null_ n ));
 DESCR("Hyperbolic cosine function");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/src/include/utils/builtins.h
----------------------------------------------------------------------
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 48de877..55ff100 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -1120,6 +1120,7 @@ extern Datum gp_remove_master_standby(PG_FUNCTION_ARGS);
 extern Datum gp_activate_standby(PG_FUNCTION_ARGS);
 
 extern Datum gp_add_segment_mirror(PG_FUNCTION_ARGS);
+extern Datum gp_remove_segment_history(PG_FUNCTION_ARGS);
 extern Datum gp_remove_segment_mirror(PG_FUNCTION_ARGS);
 extern Datum gp_add_segment(PG_FUNCTION_ARGS);
 extern Datum gp_remove_segment(PG_FUNCTION_ARGS);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1780628/tools/bin/gppylib/data/2.0.json
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/data/2.0.json b/tools/bin/gppylib/data/2.0.json
index c65e12d..019c378 100644
--- a/tools/bin/gppylib/data/2.0.json
+++ b/tools/bin/gppylib/data/2.0.json
@@ -109,8 +109,9 @@
       "CamelCaseRelationId" : "GpConfigHistoryRelationId",
       "UppercaseReltypeOid" : "GP_CONFIGURATION_HISTORY_RELTYPE_OID",
       "colh" : {
-         "dbid" : "int2",
-         "desc" : "text",
+         "description" : "text",
+         "hostname" : "text",
+         "registration_order" : "int4",
          "time" : "timestamptz"
       },
       "cols" : [
@@ -121,19 +122,24 @@
             "sqltype" : "timestamp_with_time_zone"
          },
          {
-            "colname" : "dbid",
-            "ctype" : "int2",
-            "sqltype" : "smallint"
+            "colname" : "registration_order",
+            "ctype" : "int4",
+            "sqltype" : "integer"
+         },
+         {
+            "colname" : "hostname",
+            "ctype" : "text",
+            "sqltype" : "text"
          },
          {
-            "colname" : "desc",
+            "colname" : "description",
             "ctype" : "text",
             "sqltype" : "text"
          }
       ],
       "filename" : "gp_configuration.h",
       "relid_comment_tag" : "/* relation id: 5006 - gp_configuration_history */\n",
-      "tabdef_text" : "\n   CREATE TABLE gp_configuration_history\n   with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)\n   (\n   time    timestamp with time zone,\n   dbid    smallint,\n   \"desc\"    text\n   )",
+      "tabdef_text" : "\n   CREATE TABLE gp_configuration_history\n   with (camelcase=GpConfigHistory, shared=true, oid=false, relid=5006, reltype_oid=6434, content=MASTER_ONLY)\n   (\n   time               timestamp with time zone,\n   registration_order integer,\n   hostname           text,\n   description        text\n   )",
       "tzhack" : "\"time\"",
       "with" : {
          "bootstrap" : 0,
@@ -868,8 +874,7 @@
       "UppercaseToastReltypeOid" : "GP_SEGMENT_CONFIGURATION_TOAST_RELTYPE_OID",
       "colh" : {
          "address" : "text",
-         "failed_tmpdir" : "text",
-         "failed_tmpdir_num" : "int4",
+         "description" : "text",
          "hostname" : "text",
          "port" : "int4",
          "registration_order" : "int4",
@@ -909,12 +914,7 @@
             "sqltype" : "text"
          },
          {
-            "colname" : "failed_tmpdir_num",
-            "ctype" : "int4",
-            "sqltype" : "integer"
-         },
-         {
-            "colname" : "failed_tmpdir",
+            "colname" : "description",
             "ctype" : "text",
             "sqltype" : "text"
          }
@@ -955,7 +955,7 @@
          }
       ],
       "relid_comment_tag" : "/* relation id: 5036 - gp_segment_configuration */\n",
-      "tabdef_text" : "\n   CREATE TABLE gp_segment_configuration\n   with (camelcase=GpSegmentConfig, shared=true, oid=false, relid=5036, reltype_oid=6442, toast_oid=2900, toast_index=2901, toast_reltype=2906, content=MASTER_ONLY)\n   (\n   registration_order integer    ,\n   role               \"char\"     ,\n   status             \"char\"     ,\n   port               integer    ,\n   hostname           text       ,\n   address            text       ,\n   failed_tmpdir_num  integer    ,\n   failed_tmpdir      text\n   )",
+      "tabdef_text" : "\n   CREATE TABLE gp_segment_configuration\n   with (camelcase=GpSegmentConfig, shared=true, oid=false, relid=5036, reltype_oid=6442, toast_oid=2900, toast_index=2901, toast_reltype=2906, content=MASTER_ONLY)\n   (\n   registration_order integer    ,\n   role               \"char\"     ,\n   status             \"char\"     ,\n   port               integer    ,\n   hostname           text       ,\n   address            text       ,\n   description        text\n   )",
       "with" : {
          "bootstrap" : 0,
          "camelcase" : "GpSegmentConfig",


[22/50] incubator-hawq git commit: Revert "HAWQ-533. Cursor failed, if don't allocate resource in prepare."

Posted by sh...@apache.org.
Revert "HAWQ-533. Cursor failed, if don't allocate resource in prepare."

This reverts commit 96345af194bfd3ca05227037479c9835a5e495b0.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bc065a51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bc065a51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bc065a51

Branch: refs/heads/HAWQ-459
Commit: bc065a51fe1e04c15c398a6d15c5d01424efb52f
Parents: d259eff
Author: hzhang2 <zh...@163.com>
Authored: Thu Mar 17 09:18:50 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Thu Mar 17 09:18:50 2016 +0800

----------------------------------------------------------------------
 src/backend/executor/spi.c | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bc065a51/src/backend/executor/spi.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c
index 9465de8..8e7645c 100644
--- a/src/backend/executor/spi.c
+++ b/src/backend/executor/spi.c
@@ -1146,20 +1146,6 @@ SPI_cursor_open(const char *name, SPIPlanPtr plan,
 	qtlist = copyObject(qtlist);
 	ptlist = copyObject(ptlist);
 
-	PlannedStmt* stmt = (PlannedStmt*)linitial(ptlist);
-
-	if ( (Gp_role == GP_ROLE_DISPATCH) &&
-			 (stmt->resource_parameters != NULL) )
-	{
-		/*
-		 * Now, we want to allocate resource.
-		 */
-		stmt->resource = AllocateResource(stmt->resource_parameters->life, stmt->resource_parameters->slice_size,
-				stmt->resource_parameters->iobytes, stmt->resource_parameters->max_target_segment_num,
-				stmt->resource_parameters->min_target_segment_num, stmt->resource_parameters->vol_info,
-				stmt->resource_parameters->vol_info_size);
-	}
-
 	/* If the plan has parameters, set them up */
 	if (spiplan->nargs > 0)
 	{
@@ -1860,12 +1846,12 @@ _SPI_execute_plan(_SPI_plan * plan, Datum *Values, const char *Nulls,
 				 * We only allocate resource for multiple executions of queries, NOT for utility commands.
 				 * SELECT/INSERT are supported at present.
 				 */
-				if((queryTree->commandType == CMD_SELECT) ||
-						(queryTree->commandType == CMD_INSERT))
+				if( (queryTree->commandType == CMD_SELECT) ||
+				    (queryTree->commandType == CMD_INSERT) )
 				{
-					if ((Gp_role == GP_ROLE_DISPATCH) &&
-							(stmt->resource == NULL) &&
-							(stmt->resource_parameters != NULL))
+					if ( (Gp_role == GP_ROLE_DISPATCH) &&
+					     (stmt->resource == NULL) &&
+					     (stmt->resource_parameters != NULL) )
 					{
 						SplitAllocResult *allocResult = NULL;
 


[33/50] incubator-hawq git commit: HAWQ-553. Analyze ROOTPATITION using too few vsegs.

Posted by sh...@apache.org.
HAWQ-553. Analyze ROOTPATITION using too few vsegs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/2513ad10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/2513ad10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/2513ad10

Branch: refs/heads/HAWQ-459
Commit: 2513ad108e0212b071da4301f19420c04c4db9d3
Parents: c29d647
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 10:31:58 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 10:42:44 2016 +0800

----------------------------------------------------------------------
 src/backend/commands/analyze.c | 54 ++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/2513ad10/src/backend/commands/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index dd5d5c0..32b3416 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -143,7 +143,7 @@ static void analyzeComputeAttributeStatistics(Oid relationOid,
 		float4 sampleTableRelTuples, 
 		bool mergeStats,
 		AttributeStatistics *stats);
-static List* analyzableRelations(bool rootonly);
+static List* analyzableRelations(bool rootonly, List** fullRelOids);
 static bool analyzePermitted(Oid relationOid);
 static List *analyzableAttributes(Relation candidateRelation);
 static int calculate_virtual_segment_number(List* candidateRelations);
@@ -300,6 +300,7 @@ void analyzeStatement(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 {
 	List	   			  	*lRelOids = NIL;
+	List	   			  	*lFullRelOids = NIL;
 	MemoryContext			callerContext = NULL;
 	MemoryContext 			analyzeStatementContext = NULL;
 	MemoryContext 			analyzeRelationContext = NULL;
@@ -379,7 +380,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 		/**
 		 * ANALYZE entire DB.
 		 */
-		lRelOids = analyzableRelations(stmt->rootonly);
+		lRelOids = analyzableRelations(stmt->rootonly, &lFullRelOids);
 		if (stmt->rootonly && NIL == lRelOids)
 		{
 			ereport(WARNING,
@@ -392,6 +393,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 		 * ANALYZE called by autovacuum.
 		 */
 		lRelOids = relids;
+		lFullRelOids = relids;
 	}
 	else
 	{
@@ -419,11 +421,14 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 			{
 				lRelOids = all_leaf_partition_relids(pn); /* all leaves */
 			}
+			lFullRelOids = all_leaf_partition_relids(pn);
 			lRelOids = lappend_oid(lRelOids, relationOid); /* root partition */
+			lFullRelOids = lappend_oid(lFullRelOids, relationOid); /* root partition */
 			if (optimizer_analyze_midlevel_partition)
 			{
 				lRelOids = list_concat(lRelOids, all_interior_partition_relids(pn)); /* interior partitions */
 			}
+			lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
 		}
 		else if (ps == PART_STATUS_INTERIOR) /* analyze an interior partition directly */
 		{
@@ -440,6 +445,7 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 		else
 		{
 			lRelOids = list_make1_oid(relationOid);
+			lFullRelOids = list_make1_oid(relationOid);
 		}
 	}
 
@@ -472,6 +478,17 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 	}
 
 	/**
+	 *  we use preferred_seg_num as default and
+	 *  compute target_seg_num based on data size and distributed type
+	 *  if there is no preferred_seg_num.
+	 */
+	int target_seg_num = preferred_seg_num;
+	if (target_seg_num <= 0) {
+		target_seg_num = calculate_virtual_segment_number(lFullRelOids);
+	}
+	elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
+
+	/**
 	 * We open relations with appropreciate locks
 	 */
 	List *candidateRelations = NIL;
@@ -499,17 +516,6 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
 	}
 
 	/**
-	 *  we use preferred_seg_num as default and
-	 *  compute target_seg_num based on data size and distributed type
-	 *  if there is no preferred_seg_num.
-	 */
-	int target_seg_num = preferred_seg_num;
-	if (target_seg_num <= 0) {
-		target_seg_num = calculate_virtual_segment_number(candidateRelations);
-	}
-	elog(LOG, "virtual segment number of analyze is: %d\n", target_seg_num);
-
-	/**
 	 * We allocate query resource for analyze
 	 */
 	QueryResource *resource = AllocateResource(QRL_ONCE, 1, 0, target_seg_num, target_seg_num, NULL, 0);
@@ -809,19 +815,23 @@ void analyzeStmt(VacuumStmt *stmt, List *relids, int preferred_seg_num)
  * if there is hash distributed relations exist, use the max bucket number.
  * if all relation are random, use the data size to determine vseg number.
  */
-static int calculate_virtual_segment_number(List* candidateRelations) {
+static int calculate_virtual_segment_number(List* candidateOids) {
 	ListCell* le1;
 	int vsegNumber = 1;
 	int64 totalDataSize = 0;
 	bool isHashRelationExist = false;
 	int maxHashBucketNumber = 0;
 
-	foreach (le1, candidateRelations)
+	foreach (le1, candidateOids)
 	{
-		Relation rel = (Relation)lfirst(le1);
-		if (rel ) {
+		Oid				candidateOid	  = InvalidOid;
+		candidateOid = lfirst_oid(le1);
+
+		//Relation rel = (Relation)lfirst(le1);
+		Relation rel = relation_open(candidateOid, AccessShareLock);
+		if (candidateOid > 0 ) {
 			GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
-					rel->rd_id);
+					candidateOid);
 			if(targetPolicy == NULL){
 				return GetAnalyzeVSegNumLimit();
 			}
@@ -838,6 +848,7 @@ static int calculate_virtual_segment_number(List* candidateRelations) {
 				totalDataSize += calculate_relation_size(rel);
 			}
 		}
+		relation_close(rel, AccessShareLock);
 	}
 
 	if (isHashRelationExist) {
@@ -942,7 +953,7 @@ static bool analyzePermitted(Oid relationOid)
  * Output:
  * 	List of relids
  */
-static List* analyzableRelations(bool rootonly)
+static List* analyzableRelations(bool rootonly, List **fullRelOids)
 {
 	List	   		*lRelOids = NIL;
 	cqContext		*pcqCtx;
@@ -957,6 +968,11 @@ static List* analyzableRelations(bool rootonly)
 	while (HeapTupleIsValid(tuple = caql_getnext(pcqCtx)))
 	{
 		Oid candidateOid = HeapTupleGetOid(tuple);
+		if (analyzePermitted(candidateOid)
+						&& candidateOid != StatisticRelationId)
+		{
+			*fullRelOids = lappend_oid(*fullRelOids, candidateOid);
+		}
 		if (rootonly && !rel_is_partitioned(candidateOid))
 		{
 			continue;


[30/50] incubator-hawq git commit: HAWQ-552. long latency of dispatching query resource due to wrong processing routine

Posted by sh...@apache.org.
HAWQ-552. long latency of dispatching query resource due to wrong processing routine


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3ae24a30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3ae24a30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3ae24a30

Branch: refs/heads/HAWQ-459
Commit: 3ae24a30b644c1acb939aab77931f410e6228aaf
Parents: 64663ef
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 12:40:37 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 12:40:37 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/resourcemanager.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3ae24a30/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index 45a669e..21bf0d1 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -614,11 +614,8 @@ int MainHandlerLoop(void)
         timeoutDeadResourceAllocation();
         timeoutQueuedRequest();
 
-        /* STEP 7. Generate output content to client connections. */
-        sendResponseToClients();
-
 		/*
-		 * STEP 8. Check the status of all segment nodes, mark down if hasn't got
+		 * STEP 7. Check the status of all segment nodes, mark down if hasn't got
 		 * 		   IMAlive message for a pre-defined period.
 		 */
         uint64_t curtime = gettime_microsec();
@@ -631,7 +628,7 @@ int MainHandlerLoop(void)
 			PRESPOOL->LastCheckTime = curtime;
 		}
 
-		/* STEP 9. Move all accepted GRM containers into resource pool. */
+		/* STEP 8. Move all accepted GRM containers into resource pool. */
 		moveAllAcceptedGRMContainersToResPool();
 
 		/*
@@ -657,7 +654,7 @@ int MainHandlerLoop(void)
 			setForcedReturnGRMContainerCount();
 		}
 
-		/* STEP 10. Dispatch resource to queries and send the messages out.*/
+		/* STEP 9. Dispatch resource to queries and send the messages out.*/
         if ( PRESPOOL->Segments.NodeCount > 0 && PQUEMGR->RatioCount > 0 &&
 			 PQUEMGR->toRunQueryDispatch &&
 			 PQUEMGR->ForcedReturnGRMContainerCount == 0 &&
@@ -678,6 +675,9 @@ int MainHandlerLoop(void)
 						 PQUEMGR->ForcedReturnGRMContainerCount);
         }
 
+        /* STEP 10. Generate output content to client connections. */
+        sendResponseToClients();
+
         /*
          * STEP 11. Return containers to global resource manager if there some
          * 			some idle resource.


[26/50] incubator-hawq git commit: Revert "Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted""

Posted by sh...@apache.org.
Revert "Revert "HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted""

This reverts commit 2be5899ca55b954de90aa72d00c8bd02f8f2fb83.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1897ba1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1897ba1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1897ba1d

Branch: refs/heads/HAWQ-459
Commit: 1897ba1d00135f80a8d7ff656ca8389b2be213bd
Parents: 2be5899
Author: hubertzhang <hz...@pivotal.io>
Authored: Thu Mar 17 15:37:09 2016 +0800
Committer: hubertzhang <hz...@pivotal.io>
Committed: Thu Mar 17 15:37:09 2016 +0800

----------------------------------------------------------------------
 src/backend/resourcemanager/include/errorcode.h   |  1 +
 .../resourcebroker/resourcebroker_LIBYARN.c       | 18 ++++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1897ba1d/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index ca338a8..de08fff 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,6 +162,7 @@ enum DRM_ERROR_CODE {
 	RESBROK_WRONG_GLOB_MGR_ADDRESS,
 	RESBROK_WRONG_GLOB_MGR_QUEUE,
 	RESBROK_WRONG_GLOB_MGR_APPNAME,
+	RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
 	RESBROK_PIPE_ERROR,
 	RESBROK_NOMORE_RESOURCE_IN_GRM,
 	RESBROK_WRONG_MESSAGE_ID,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1897ba1d/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index cdbe0d5..b8d205e 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
 	}
 
 	elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
-			  "maximum capacity is %lf.",
+			  "maximum queue capacity is %lf.",
 			  response.MachineCount,
 			  response.QueueMaxCapacity);
 
@@ -631,6 +631,19 @@ int handleRB2RM_ClusterReport(void)
 	 * TILL NOW, the whole message content is received.
 	 */
 
+	if ( res == FUNC_RETURN_OK )
+	{
+		/* Check if the YARN resource queue report is valid, i.e. maximum
+		 * capacity and capacity are all greater than 0.
+		 */
+		if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
+		{
+			elog(WARNING, "YARN mode resource broker got invalid cluster report");
+			res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
+
+		}
+	}
+
 	/* If something wrong, no need to keep the received content, free them. */
 	if ( res != FUNC_RETURN_OK )
 	{
@@ -1176,6 +1189,7 @@ void RB_LIBYARN_handleError(int errorcode)
 	}
 	else
 	{
-		Assert(errorcode == FUNC_RETURN_OK);
+		Assert(errorcode == FUNC_RETURN_OK ||
+			   errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
 	}
 }


[46/50] incubator-hawq git commit: HAWQ-563. Restore guc hawq_rm_nvseg_variance_amon_seg_respool_limit and fix its bug when resource manager resets resource

Posted by sh...@apache.org.
HAWQ-563. Restore guc hawq_rm_nvseg_variance_amon_seg_respool_limit and fix its bug when resource manager resets resource


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/802d537c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/802d537c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/802d537c

Branch: refs/heads/HAWQ-459
Commit: 802d537cc8e19618fd2553608746ae595d8f8ec2
Parents: 0bb22dc
Author: YI JIN <yj...@pivotal.io>
Authored: Mon Mar 21 15:47:04 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Mon Mar 21 15:47:04 2016 +1100

----------------------------------------------------------------------
 src/backend/cdb/cdbvars.c                         |  2 ++
 .../resourcemanager/include/resourcepool.h        |  1 +
 src/backend/resourcemanager/resourcemanager.c     |  1 +
 src/backend/resourcemanager/resourcepool.c        | 18 ++++++++++++++++--
 src/backend/utils/misc/guc.c                      | 11 +++++++++++
 5 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index a2476fd..b4a9f3d 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -390,6 +390,8 @@ double	rm_regularize_io_factor;
 double	rm_regularize_usage_factor;
 double	rm_regularize_nvseg_factor;
 
+int		rm_nvseg_variance_among_seg_respool_limit;
+
 /* Greenplum Database Experimental Feature GUCs */
 int         gp_distinct_grouping_sets_threshold = 32;
 bool		gp_enable_explain_allstat = FALSE;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/include/resourcepool.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h
index f965c43..08ce305 100644
--- a/src/backend/resourcemanager/include/resourcepool.h
+++ b/src/backend/resourcemanager/include/resourcepool.h
@@ -633,6 +633,7 @@ int getOrderedResourceAllocTreeIndexByRatio(uint32_t ratio, BBST *tree);
 void setAllSegResourceGRMUnhandled(void);
 
 void resetAllSegmentsGRMContainerFailAllocCount(void);
+void resetAllSegmentsNVSeg(void);
 
 struct RB_GRMContainerStatData
 {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c
index c585d40..c7215be 100644
--- a/src/backend/resourcemanager/resourcemanager.c
+++ b/src/backend/resourcemanager/resourcemanager.c
@@ -580,6 +580,7 @@ int MainHandlerLoop(void)
 			refreshMemoryCoreRatioLevelUsage(gettime_microsec());
 
 			resetAllSegmentsGRMContainerFailAllocCount();
+			resetAllSegmentsNVSeg();
 
 			/* Check if can resume using new available global resource manager.*/
 			if ( cleanedAllGRMContainers() )

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcepool.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c
index acc8dbe..3a8d72b 100644
--- a/src/backend/resourcemanager/resourcepool.c
+++ b/src/backend/resourcemanager/resourcepool.c
@@ -1650,6 +1650,20 @@ void resetAllSegmentsGRMContainerFailAllocCount(void)
 	freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
 }
 
+void resetAllSegmentsNVSeg(void)
+{
+	List 	 *allsegres = NULL;
+	ListCell *cell		= NULL;
+	getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres);
+
+	foreach(cell, allsegres)
+	{
+		SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value);
+		segres->NVSeg = 0;
+	}
+	freePAIRRefList(&(PRESPOOL->Segments), &allsegres);
+}
+
 /*
  * Check index to get host id based on host name string.
  */
@@ -2556,7 +2570,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 		freePAIRRefList(&(PRESPOOL->Segments), &ressegl);
 		Assert(minnvseg <= maxnvseg);
 
-		if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_limit )
+		if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_respool_limit )
 		{
 			elog(LOG, "Reject virtual segment allocation based on data "
 					  "locality information. After tentative allocation "
@@ -2565,7 +2579,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t 	 nodecount,
 					  "is %d, tolerated difference limit is %d.",
 					  maxnvseg,
 					  minnvseg,
-					  rm_nvseg_variance_among_seg_limit);
+					  rm_nvseg_variance_among_seg_respool_limit);
 
 			/* Return the allocated resource. */
 			List 	 *vsegcntlist = NULL;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 56909c9..a881b31 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6557,6 +6557,17 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
+		{"hawq_rm_nvseg_variance_amon_seg_respool_limit", PGC_POSTMASTER, RESOURCES_MGM,
+			gettext_noop("the variance of vseg number in each segment that resource manager "
+						 "should tolerate at most in resource pool when choosing segments "
+						 "based on data locality reference."),
+			NULL
+		},
+		&rm_nvseg_variance_among_seg_respool_limit,
+		2, 0, 65535, NULL, NULL
+	},
+
+	{
 		{"hawq_rm_container_batch_limit", PGC_POSTMASTER, RESOURCES_MGM,
 			gettext_noop("the batch process limit for global resource manager containers."),
 			NULL


[41/50] incubator-hawq git commit: Revert "HAWQ-532. Optimise vseg number for copy to statement."

Posted by sh...@apache.org.
Revert "HAWQ-532. Optimise vseg number for copy to statement."


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/86d9b0c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/86d9b0c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/86d9b0c0

Branch: refs/heads/HAWQ-459
Commit: 86d9b0c03de349398ddb0337f7ac5dac3b58d2b9
Parents: 36571bc
Author: hzhang2 <zh...@163.com>
Authored: Fri Mar 18 14:22:29 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Fri Mar 18 14:22:29 2016 +0800

----------------------------------------------------------------------
 src/backend/commands/copy.c | 97 ++++------------------------------------
 1 file changed, 8 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/86d9b0c0/src/backend/commands/copy.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e37fb83..bcd5384 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -91,11 +91,6 @@
 #include "postmaster/autovacuum.h"
 #include "cdb/dispatcher.h"
 
-/*
- * in dbsize.c
- */
-extern int64 calculate_relation_size(Relation rel);
-
 /* DestReceiver for COPY (SELECT) TO */
 typedef struct
 {
@@ -142,7 +137,6 @@ static void copy_in_error_callback(void *arg);
 static void CopyInitPartitioningState(EState *estate);
 static void CopyInitDataParser(CopyState cstate);
 static bool CopyCheckIsLastLine(CopyState cstate);
-static int calculate_virtual_segment_number(List* candidateRelations);
 
 /* ==========================================================================
  * The follwing macros aid in major refactoring of data processing code (in
@@ -1562,16 +1556,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 
 			target_policy = GpPolicyFetch(CurrentMemoryContext, relid);
 			Assert(target_policy);
-			/*
-			 * For hash table we use table bucket number to request vsegs
-			 * For random table, we use a fixed GUC value to request vsegs.
-			 */
-			if(target_policy->nattrs > 0){
-				target_segment_num = target_policy->bucketnum;
-			}
-			else{
-				target_segment_num = hawq_rm_nvseg_for_copy_from_perquery;
-			}
+			target_segment_num = target_policy->bucketnum;
 			pfree(target_policy);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
@@ -1764,63 +1749,6 @@ DoCopy(const CopyStmt *stmt, const char *queryString)
 }
 
 /*
- * calculate virtual segment number for copy statement.
- * if there is hash distributed relations exist, use the max bucket number.
- * if all relation are random, use the data size to determine vseg number.
- */
-static int calculate_virtual_segment_number(List* candidateOids) {
-	ListCell* le1;
-	int vsegNumber = 1;
-	int64 totalDataSize = 0;
-	bool isHashRelationExist = false;
-	int maxHashBucketNumber = 0;
-
-	foreach (le1, candidateOids)
-	{
-		Oid				candidateOid	  = InvalidOid;
-		candidateOid = lfirst_oid(le1);
-
-		//Relation rel = (Relation)lfirst(le1);
-		Relation rel = relation_open(candidateOid, AccessShareLock);
-		if (candidateOid > 0 ) {
-			GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext,
-					candidateOid);
-			if(targetPolicy == NULL){
-				return GetAnalyzeVSegNumLimit();
-			}
-			if (targetPolicy->nattrs > 0) {
-				isHashRelationExist = true;
-				if(maxHashBucketNumber < targetPolicy->bucketnum){
-					maxHashBucketNumber = targetPolicy->bucketnum;
-				}
-			}
-			/*
-			 * if no hash relation, we calculate the data size of all the relations.
-			 */
-			if (!isHashRelationExist) {
-				totalDataSize += calculate_relation_size(rel);
-			}
-		}
-		relation_close(rel, AccessShareLock);
-	}
-
-	if (isHashRelationExist) {
-		vsegNumber = maxHashBucketNumber;
-	} else {
-		/*we allocate one virtual segment for each 128M data */
-		totalDataSize >>= 27;
-		vsegNumber = totalDataSize + 1;
-	}
-	Assert(vsegNumber > 0);
-	/*vsegNumber should be less than GetUtilPartitionNum*/
-	if(vsegNumber > GetQueryVsegNum()){
-		vsegNumber = GetQueryVsegNum();
-	}
-
-	return vsegNumber;
-}
-
-/*
  * This intermediate routine exists mainly to localize the effects of setjmp
  * so we don't need to plaster a lot of variables with "volatile".
  */
@@ -1881,22 +1809,13 @@ DoCopyTo(CopyState cstate)
 		 */
 		if (Gp_role == GP_ROLE_DISPATCH && cstate->rel && cstate->rel->rd_cdbpolicy)
 		{
+			GpPolicy *target_policy = NULL;
 			int target_segment_num = 0;
-			/*
-			 * copy hash table use table bucket number
-			 * copy random table use table size.
-			 */
-			PartitionNode *pn = get_parts(cstate->rel->rd_id, 0 /*level*/ ,
-													0 /*parent*/, false /* inctemplate */, CurrentMemoryContext, true /*includesubparts*/);
-			List		*lFullRelOids = NIL;
-			if(pn){
-				lFullRelOids = all_leaf_partition_relids(pn);
-				lFullRelOids = list_concat(lFullRelOids, all_interior_partition_relids(pn)); /* interior partitions */
-			}
-			lFullRelOids = lappend_oid(lFullRelOids, cstate->rel->rd_id);
 
-			target_segment_num = calculate_virtual_segment_number(lFullRelOids);
-			elog(LOG, "virtual segment number of copy to is: %d\n", target_segment_num);
+			target_policy = GpPolicyFetch(CurrentMemoryContext, cstate->rel->rd_id);
+			Assert(target_policy);
+			target_segment_num = target_policy->bucketnum;
+			pfree(target_policy);
 
 			cstate->resource = AllocateResource(QRL_ONCE, 1, 1, target_segment_num, target_segment_num,NULL,0);
 			CopyToDispatch(cstate);
@@ -4281,7 +4200,7 @@ CopyFrom(CopyState cstate)
 					if (cstate->oids && file_has_oids)
 						MemTupleSetOid(tuple, resultRelInfo->ri_aoInsertDesc->mt_bind, loaded_oid);
 				}
-				else if (relstorage == RELSTORAGE_PARQUET)
+				else if ((relstorage == RELSTORAGE_PARQUET))
 				{
                     tuple = NULL;
 				}
@@ -4312,7 +4231,7 @@ CopyFrom(CopyState cstate)
 				{
 					HeapTuple	newtuple;
 
-					if(relstorage == RELSTORAGE_PARQUET)
+					if((relstorage == RELSTORAGE_PARQUET))
 					{
 						Assert(!tuple);
 						elog(ERROR, "triggers are not supported on tables that use column-oriented storage");


[09/50] incubator-hawq git commit: HAWQ-540. Change output.replace-datanode-on-failure to true by default.

Posted by sh...@apache.org.
HAWQ-540. Change output.replace-datanode-on-failure to true by default.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a7ad3753
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a7ad3753
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a7ad3753

Branch: refs/heads/HAWQ-459
Commit: a7ad3753c064d6a7f8c4c28f691fc346d7d68c96
Parents: d178062
Author: zhenglin tao <zh...@gmail.com>
Authored: Tue Mar 15 10:12:36 2016 +0800
Committer: zhenglin tao <zh...@gmail.com>
Committed: Tue Mar 15 10:12:36 2016 +0800

----------------------------------------------------------------------
 src/backend/utils/misc/etc/hdfs-client.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a7ad3753/src/backend/utils/misc/etc/hdfs-client.xml
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/etc/hdfs-client.xml b/src/backend/utils/misc/etc/hdfs-client.xml
index c0f395b..cf51e1a 100644
--- a/src/backend/utils/misc/etc/hdfs-client.xml
+++ b/src/backend/utils/misc/etc/hdfs-client.xml
@@ -236,7 +236,7 @@ HA -->
 	<!-- output client configuration -->
 	<property>
 		<name>output.replace-datanode-on-failure</name>
-		<value>false</value>
+		<value>true</value>
 		<description>
 		whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
 		</description>


[50/50] incubator-hawq git commit: Merge remote-tracking branch 'origin/master' into HAWQ-459

Posted by sh...@apache.org.
Merge remote-tracking branch 'origin/master' into HAWQ-459


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/55be00ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/55be00ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/55be00ce

Branch: refs/heads/HAWQ-459
Commit: 55be00cee13cf1294ec75c7a50a928af735c2d4d
Parents: b1198ed 76dc631
Author: Shivram Mani <sh...@gmail.com>
Authored: Mon Mar 21 15:01:10 2016 -0700
Committer: Shivram Mani <sh...@gmail.com>
Committed: Mon Mar 21 15:01:10 2016 -0700

----------------------------------------------------------------------
 configure                                       |  27 -
 configure.in                                    |  27 -
 .../hawq/mapreduce/TPCHClusterTester.java       |   4 +-
 src/backend/Makefile                            |   8 +-
 src/backend/catalog/gp_toolkit.sql.in           | 223 -----
 src/backend/catalog/system_views.sql            |  14 +-
 src/backend/cdb/cdbcopy.c                       |   2 +-
 src/backend/cdb/cdbdatalocality.c               | 338 ++++---
 src/backend/cdb/cdbdispatchresult.c             |  28 +-
 src/backend/cdb/cdbfilesplit.c                  |  67 +-
 src/backend/cdb/cdbmutate.c                     |   2 +-
 src/backend/cdb/cdbvars.c                       |   3 +
 src/backend/cdb/dispatcher.c                    |  18 +-
 src/backend/cdb/dispatcher_mgt.c                |   9 +-
 src/backend/cdb/executormgr.c                   |  32 +-
 src/backend/cdb/motion/ic_common.c              |   7 +-
 src/backend/cdb/workermgr.c                     |  10 +
 src/backend/commands/analyze.c                  |  54 +-
 src/backend/commands/copy.c                     |  97 +-
 src/backend/commands/explain.c                  |   2 +-
 src/backend/commands/prepare.c                  |   2 +-
 src/backend/commands/tablecmds.c                |  50 +-
 src/backend/executor/execDML.c                  |  99 +-
 src/backend/executor/execMain.c                 |  13 +-
 src/backend/executor/execUtils.c                |   2 +-
 src/backend/executor/nodeDML.c                  |   2 +-
 src/backend/executor/spi.c                      |  71 +-
 src/backend/gpopt/gpopt.mk                      |  18 +-
 src/backend/gpopt/ivy.xml                       |   2 +-
 .../gpopt/translate/CTranslatorDXLToPlStmt.cpp  |   2 +
 .../translate/CTranslatorRelcacheToDXL.cpp      |   9 +
 src/backend/gpopt/utils/COptTasks.cpp           |   4 +-
 src/backend/nodes/copyfuncs.c                   |   1 +
 src/backend/nodes/outfast.c                     |   1 +
 src/backend/nodes/outfuncs.c                    |   1 +
 src/backend/nodes/readfast.c                    |   1 +
 src/backend/optimizer/plan/planner.c            |  60 +-
 src/backend/optimizer/util/clauses.c            |  11 +-
 src/backend/optimizer/util/relnode.c            |   2 +-
 src/backend/optimizer/util/walkers.c            |   1 +
 src/backend/parser/analyze.c                    |  45 +-
 src/backend/parser/gram.y                       |   9 +-
 src/backend/postmaster/identity.c               |   2 +-
 src/backend/postmaster/postmaster.c             |   2 -
 .../communication/rmcomm_AsyncComm.c            | 317 +++---
 .../communication/rmcomm_Connect.c              |  11 +-
 .../communication/rmcomm_Message.c              |   6 +-
 .../communication/rmcomm_MessageHandler.c       |  15 +-
 .../communication/rmcomm_MessageServer.c        | 134 +--
 .../communication/rmcomm_QD2RM.c                | 149 +--
 .../communication/rmcomm_RM2RMSEG.c             |  42 +-
 .../communication/rmcomm_RMSEG2RM.c             |   6 +-
 .../communication/rmcomm_SyncComm.c             | 108 +-
 src/backend/resourcemanager/conntrack.c         |  90 +-
 .../include/communication/rmcomm_AsyncComm.h    |  36 +-
 .../include/communication/rmcomm_Connect.h      |   1 -
 .../communication/rmcomm_MessageServer.h        |  12 +-
 .../include/communication/rmcomm_SyncComm.h     |   9 -
 src/backend/resourcemanager/include/conntrack.h |  11 +-
 src/backend/resourcemanager/include/errorcode.h |   1 +
 .../resourcemanager/include/resourcepool.h      |  47 +-
 .../include/utils/network_utils.h               |  40 +-
 src/backend/resourcemanager/requesthandler.c    | 178 ++--
 .../resourcemanager/requesthandler_RMSEG.c      |   6 +-
 .../resourcebroker/resourcebroker_LIBYARN.c     |  98 +-
 .../resourcebroker_LIBYARN_proc.c               |   3 +-
 src/backend/resourcemanager/resourcemanager.c   | 236 ++---
 .../resourcemanager/resourcemanager_RMSEG.c     |   1 -
 src/backend/resourcemanager/resourcepool.c      | 986 +++++++++++++------
 src/backend/resourcemanager/resqueuemanager.c   |  42 +-
 .../resourcemanager/utils/network_utils.c       | 407 +++++---
 src/backend/tcop/postgres.c                     |   7 +-
 src/backend/utils/adt/test/Makefile             |  42 +
 src/backend/utils/adt/test/varlena_test.c       | 251 +++++
 src/backend/utils/adt/varlena.c                 | 334 +++++--
 src/backend/utils/cache/relcache.c              |   2 +-
 src/backend/utils/gp/segadmin.c                 |   3 +-
 src/backend/utils/misc/etc/gpcheck.cnf          |   2 +-
 src/backend/utils/misc/etc/hdfs-client.xml      |   2 +-
 src/backend/utils/misc/faultinjector.c          |   9 +-
 src/backend/utils/misc/guc.c                    |  51 +-
 src/bin/gpcheckhdfs/Makefile                    |   2 +-
 src/bin/gpfilesystem/hdfs/Makefile              |   2 +-
 src/include/access/filesplit.h                  |   2 +-
 src/include/catalog/gp_configuration.h          |  27 +-
 src/include/catalog/gp_segment_config.h         |  11 +-
 src/include/catalog/pg_proc.h                   |   5 +-
 src/include/cdb/cdbdatalocality.h               |  21 +-
 src/include/cdb/cdbvars.h                       |   3 +
 src/include/executor/execDML.h                  |   3 +-
 src/include/executor/spi.h                      |   7 +
 src/include/gpopt/translate/CTranslatorUtils.h  |   4 -
 src/include/nodes/execnodes.h                   |   7 +-
 src/include/nodes/plannodes.h                   |   2 +
 src/include/optimizer/planner.h                 |   4 +
 src/include/postmaster/identity.h               |   2 +-
 src/include/utils/builtins.h                    |   1 +
 src/include/utils/faultinjector.h               |   2 +
 src/include/utils/guc.h                         |   1 +
 .../expected/create_table_distribution.out      | 102 +-
 src/test/regress/expected/function.out          |  16 +
 .../regress/expected/function_optimizer.out     |  16 +
 src/test/regress/expected/goh_partition.out     | 126 +++
 .../expected/gpsql_alter_table_optimizer.out    |   2 +-
 src/test/regress/expected/insert_optimizer.out  |   2 +-
 src/test/regress/output/exttab1.source          |   6 +-
 .../regress/output/exttab1_optimizer.source     |   6 +-
 .../regress/sql/create_table_distribution.sql   |  41 +-
 src/test/regress/sql/function.sql               |  12 +
 src/test/regress/sql/goh_partition.sql          |  68 ++
 tools/bin/gppylib/data/2.0.json                 |  32 +-
 tools/bin/gppylib/programs/clsInjectFault.py    |   5 +-
 tools/bin/hawq_ctl                              | 117 ++-
 tools/bin/hawqextract                           |  32 +-
 tools/bin/hawqpylib/HAWQ_HELP.py                |   2 +-
 tools/bin/hawqpylib/hawqlib.py                  | 129 ++-
 tools/bin/hawqpylib/programs/clsInjectFault.py  |   5 +-
 tools/bin/lib/hawqinit.sh                       |  66 +-
 118 files changed, 3884 insertions(+), 2084 deletions(-)
----------------------------------------------------------------------



[17/50] incubator-hawq git commit: HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted

Posted by sh...@apache.org.
HAWQ-547. When HAWQ RM does not have valid YARN resource queue status received the cluster report is not accepted


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5fda1826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5fda1826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5fda1826

Branch: refs/heads/HAWQ-459
Commit: 5fda1826aff4780b20e24db5210f79f867ef39ae
Parents: 8e38e84
Author: YI JIN <yj...@pivotal.io>
Authored: Wed Mar 16 13:52:37 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Wed Mar 16 13:52:37 2016 +1100

----------------------------------------------------------------------
 src/backend/resourcemanager/include/errorcode.h   |  1 +
 .../resourcebroker/resourcebroker_LIBYARN.c       | 18 ++++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5fda1826/src/backend/resourcemanager/include/errorcode.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/errorcode.h b/src/backend/resourcemanager/include/errorcode.h
index ca338a8..de08fff 100644
--- a/src/backend/resourcemanager/include/errorcode.h
+++ b/src/backend/resourcemanager/include/errorcode.h
@@ -162,6 +162,7 @@ enum DRM_ERROR_CODE {
 	RESBROK_WRONG_GLOB_MGR_ADDRESS,
 	RESBROK_WRONG_GLOB_MGR_QUEUE,
 	RESBROK_WRONG_GLOB_MGR_APPNAME,
+	RESBROK_WRONG_GLOB_MGR_QUEUEREPORT,
 	RESBROK_PIPE_ERROR,
 	RESBROK_NOMORE_RESOURCE_IN_GRM,
 	RESBROK_WRONG_MESSAGE_ID,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5fda1826/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
index cdbe0d5..b8d205e 100644
--- a/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
+++ b/src/backend/resourcemanager/resourcebroker/resourcebroker_LIBYARN.c
@@ -568,7 +568,7 @@ int handleRB2RM_ClusterReport(void)
 	}
 
 	elog(LOG, "YARN mode resource broker got cluster report having %d host(s), "
-			  "maximum capacity is %lf.",
+			  "maximum queue capacity is %lf.",
 			  response.MachineCount,
 			  response.QueueMaxCapacity);
 
@@ -631,6 +631,19 @@ int handleRB2RM_ClusterReport(void)
 	 * TILL NOW, the whole message content is received.
 	 */
 
+	if ( res == FUNC_RETURN_OK )
+	{
+		/* Check if the YARN resource queue report is valid, i.e. maximum
+		 * capacity and capacity are all greater than 0.
+		 */
+		if ( response.QueueCapacity <= 0 || response.QueueMaxCapacity <= 0 )
+		{
+			elog(WARNING, "YARN mode resource broker got invalid cluster report");
+			res = RESBROK_WRONG_GLOB_MGR_QUEUEREPORT;
+
+		}
+	}
+
 	/* If something wrong, no need to keep the received content, free them. */
 	if ( res != FUNC_RETURN_OK )
 	{
@@ -1176,6 +1189,7 @@ void RB_LIBYARN_handleError(int errorcode)
 	}
 	else
 	{
-		Assert(errorcode == FUNC_RETURN_OK);
+		Assert(errorcode == FUNC_RETURN_OK ||
+			   errorcode == RESBROK_WRONG_GLOB_MGR_QUEUEREPORT);
 	}
 }


[14/50] incubator-hawq git commit: HAWQ-499: Fix IPv6 on OSX > 10.11

Posted by sh...@apache.org.
HAWQ-499: Fix IPv6 on OSX > 10.11

- OSX IPv6 addresses are currently fetched with a command that
  doesn't exist on OSX for versions < 10.11.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca2a7708
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca2a7708
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca2a7708

Branch: refs/heads/HAWQ-459
Commit: ca2a77088def9d22419cf0cf9db1fa01826e0cb0
Parents: c72d066
Author: Jacob Frank <jf...@pivotal.io>
Authored: Tue Mar 8 16:41:31 2016 -0800
Committer: rlei <rl...@pivotal.io>
Committed: Tue Mar 15 16:29:06 2016 +0800

----------------------------------------------------------------------
 tools/bin/lib/hawqinit.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca2a7708/tools/bin/lib/hawqinit.sh
----------------------------------------------------------------------
diff --git a/tools/bin/lib/hawqinit.sh b/tools/bin/lib/hawqinit.sh
index af5635a..dab0b34 100755
--- a/tools/bin/lib/hawqinit.sh
+++ b/tools/bin/lib/hawqinit.sh
@@ -190,7 +190,7 @@ LOAD_GP_TOOLKIT () {
 }
 
 get_master_ipv6_addresses() {
-    if [ "${distro_based_on}" = "Mac" ] && [ "${distro_version:0:5}" = "10.11" ]; then
+    if [ "${distro_based_on}" = "Mac" ]; then
         MASTER_IPV6_LOCAL_ADDRESS_ALL=(`${IFCONFIG} | ${GREP} inet6 | ${AWK} '{print $2}' | cut -d'%' -f1`)
     else
         MASTER_IPV6_LOCAL_ADDRESS_ALL=(`ip -6 address show |${GREP} inet6|${AWK} '{print $2}' |cut -d'/' -f1`)


[06/50] incubator-hawq git commit: HAWQ-528. Reset gp_connections_per_thread for dispatcher guc range from 1 to 512, 0 marks as invalid.

Posted by sh...@apache.org.
HAWQ-528. Reset gp_connections_per_thread for dispatcher guc range from 1 to 512, 0 marks as invalid.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4b53168b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4b53168b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4b53168b

Branch: refs/heads/HAWQ-459
Commit: 4b53168b537832f03aa6b4160a78afb63f34fbec
Parents: d5e7b92
Author: Lili Ma <lm...@pivotal.io>
Authored: Mon Mar 14 17:39:31 2016 +0800
Committer: Lili Ma <lm...@pivotal.io>
Committed: Tue Mar 15 09:38:40 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/dispatcher.c     | 7 +------
 src/backend/cdb/dispatcher_mgt.c | 9 +--------
 src/backend/utils/misc/guc.c     | 2 +-
 3 files changed, 3 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 57ccdf4..20e97ab 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1656,12 +1656,7 @@ dispatcher_compute_threads_num(DispatchData *data)
 	/* TODO: decide the groups(threads) number. */
 	if (query_executors_num == 0)
 		threads_num = 1;
-	if (executors_num_per_thread == 0)
-	{
-		threads_num = query_executors_num;
-		executors_num_per_thread = 1;
-	}
-	else if (executors_num_per_thread > query_executors_num)
+	if (executors_num_per_thread > query_executors_num)
 		threads_num = 1;
 	else
 		threads_num = (query_executors_num / executors_num_per_thread) + ((query_executors_num % executors_num_per_thread == 0) ? 0 : 1); 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/cdb/dispatcher_mgt.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher_mgt.c b/src/backend/cdb/dispatcher_mgt.c
index 2920249..d0fd7e7 100644
--- a/src/backend/cdb/dispatcher_mgt.c
+++ b/src/backend/cdb/dispatcher_mgt.c
@@ -449,14 +449,7 @@ dispmgt_create_concurrent_connect_state(List *executors, int executors_num_per_t
 	int		i, j;
 
 	/* Compute threads_num */
-	if (executors_num == 0)
-		return NIL;
-	else if (executors_num_per_thread == 0)
-	{
-		threads_num = executors_num;
-		executors_num_per_thread = 1;
-	}
-	else if (executors_num_per_thread > executors_num)
+	if (executors_num_per_thread > executors_num)
 		threads_num = 1;
 	else
 		threads_num = (executors_num / executors_num_per_thread) + ((executors_num % executors_num_per_thread == 0) ? 0 : 1); 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/4b53168b/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5b04054..aac7192 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5647,7 +5647,7 @@ static struct config_int ConfigureNamesInt[] =
 			GUC_NOT_IN_SAMPLE | GUC_GPDB_ADDOPT
 		},
 		&gp_connections_per_thread,
-		512, 0, INT_MAX, assign_gp_connections_per_thread, show_gp_connections_per_thread
+		512, 1, INT_MAX, assign_gp_connections_per_thread, show_gp_connections_per_thread
 	},
 
 	{


[45/50] incubator-hawq git commit: HAWQ-559. Fix QD hang when QE is immediately killed after created

Posted by sh...@apache.org.
HAWQ-559. Fix QD hang when QE is immediately killed after created


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/0bb22dce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/0bb22dce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/0bb22dce

Branch: refs/heads/HAWQ-459
Commit: 0bb22dce146cbd34507293b32f5603a6dc706aec
Parents: 6e88cba
Author: Lili Ma <ic...@gmail.com>
Authored: Fri Mar 18 16:49:06 2016 +0800
Committer: Lili Ma <ic...@gmail.com>
Committed: Mon Mar 21 09:49:06 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/dispatcher.c  |  2 ++
 src/backend/cdb/executormgr.c | 20 +++++++++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/dispatcher.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/dispatcher.c b/src/backend/cdb/dispatcher.c
index 5d7105c..55a7823 100644
--- a/src/backend/cdb/dispatcher.c
+++ b/src/backend/cdb/dispatcher.c
@@ -1240,6 +1240,8 @@ dispatch_run(DispatchData *data)
 	 * Only after we have the executors, we can serialize the state. Or we
 	 * don't know the executor listening address.
 	 */
+	CHECK_FOR_INTERRUPTS();
+
 	dispatcher_serialize_state(data);
 	dispatcher_serialize_query_resource(data);
 	dispatcher_set_state_run(data);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/0bb22dce/src/backend/cdb/executormgr.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 47f5bbe..2bc5059 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -364,7 +364,19 @@ executormgr_validate_conn(PGconn *conn)
 {
   if (conn == NULL)
     return false;
-  return dispatch_validate_conn(conn->sock);
+  if (!dispatch_validate_conn(conn->sock))
+  {
+    printfPQExpBuffer(&conn->errorMessage,
+              libpq_gettext(
+                  "server closed the connection unexpectedly\n"
+             "\tThis probably means the server terminated abnormally\n"
+                 "\tbefore or while processing the request.\n"));
+    conn->status = CONNECTION_BAD;
+    closesocket(conn->sock);
+    conn->sock = -1;
+    return false;
+  }
+  return true;
 }
 
 /*
@@ -408,7 +420,7 @@ executormgr_dispatch_and_run(struct DispatchData *data, QueryExecutor *executor)
 	DispatchCommandQueryParms	*parms = dispatcher_get_QueryParms(data);
 
 	if (!executormgr_is_dispatchable(executor))
-		return false;
+	  goto error;
 
 	TIMING_BEGIN(executor->time_dispatch_begin);
 	query = PQbuildGpQueryString(parms->strCommand, parms->strCommandlen,
@@ -573,10 +585,8 @@ executormgr_catch_error(QueryExecutor *executor)
 
 	msg = PQerrorMessage(conn);
 
-	if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode == 0)){
+	if (msg && (strcmp("", msg) != 0) && (executor->refResult->errcode == 0)) {
 	  errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
-	  if (errCode == ERRCODE_GP_INTERCONNECTION_ERROR)
-	    errCode = ERRCODE_GP_INTERCONNECTION_ERROR;
 	}
 
 	PQExpBufferData selfDesc;


[21/50] incubator-hawq git commit: HAWQ-548. Fix wrong unit test case for ip address parsing of interconnect

Posted by sh...@apache.org.
HAWQ-548. Fix wrong unit test case for ip address parsing of interconnect


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d259efff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d259efff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d259efff

Branch: refs/heads/HAWQ-459
Commit: d259efffbce3d9b1eb4e68646632d6419cf2c745
Parents: 1a75a2a
Author: Lili Ma <ic...@gmail.com>
Authored: Wed Mar 16 13:40:05 2016 +0800
Committer: Lili Ma <ic...@gmail.com>
Committed: Wed Mar 16 14:18:49 2016 +0800

----------------------------------------------------------------------
 src/test/regress/expected/function.out | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d259efff/src/test/regress/expected/function.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/function.out b/src/test/regress/expected/function.out
index d493ed5..bda1ded 100644
--- a/src/test/regress/expected/function.out
+++ b/src/test/regress/expected/function.out
@@ -881,7 +881,6 @@ DROP FUNCTION outer(int);
 DROP FUNCTION inner(int);
 -- TEARDOWN
 DROP TABLE foo;
-
 -- HAWQ-510
 drop table if exists testEntryDB;
 NOTICE:  table "testentrydb" does not exist, skipping
@@ -895,3 +894,6 @@ where  t1.value=t2.value;
 -----+-------
    1 |     0
    2 |     0
+(2 rows)
+
+drop table testEntryDB;


[38/50] incubator-hawq git commit: HAWQ-557. Set connection pool connections always NODELAY and KEEPALIVE

Posted by sh...@apache.org.
HAWQ-557. Set connection pool connections always NODELAY and KEEPALIVE


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e82ac3f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e82ac3f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e82ac3f1

Branch: refs/heads/HAWQ-459
Commit: e82ac3f17319dcc630b467868491089a189aeeec
Parents: f38ad11
Author: YI JIN <yj...@pivotal.io>
Authored: Fri Mar 18 13:58:18 2016 +1100
Committer: YI JIN <yj...@pivotal.io>
Committed: Fri Mar 18 13:58:18 2016 +1100

----------------------------------------------------------------------
 .../communication/rmcomm_AsyncComm.c            |  6 +++++
 .../communication/rmcomm_Connect.c              |  5 ++++
 .../communication/rmcomm_QD2RM.c                | 27 ++++++++++++++++++++
 .../include/utils/network_utils.h               |  1 +
 .../resourcemanager/utils/network_utils.c       | 20 +++++++++++++++
 5 files changed, 59 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c b/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
index 0084edc..181cc86 100644
--- a/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
+++ b/src/backend/resourcemanager/communication/rmcomm_AsyncComm.c
@@ -718,6 +718,12 @@ int registerAsyncConnectionFileDesc(const char				*address,
 	sockres = connect(fd, (struct sockaddr *)&server_addr, sizeof(server_addr));
 	if ( sockres == 0 )
 	{
+		if ( setConnectionLongTermNoDelay(fd) != FUNC_RETURN_OK )
+		{
+			close(fd);
+			res = UTIL_NETWORK_FAIL_CONNECT;
+			goto exit;
+		}
 		/*
 		 * New connection is created. Suppose domain socket and local socket
 		 * connection can be done now. Register a normal client FD in poll() to

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_Connect.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_Connect.c b/src/backend/resourcemanager/communication/rmcomm_Connect.c
index fd916da..6c58a5c 100644
--- a/src/backend/resourcemanager/communication/rmcomm_Connect.c
+++ b/src/backend/resourcemanager/communication/rmcomm_Connect.c
@@ -68,6 +68,11 @@ void WriteReadyHandler_Connect(AsyncCommBuffer buffer)
 		shouldclose = true;
 	}
 
+	if ( setConnectionLongTermNoDelay(buffer->FD) != FUNC_RETURN_OK )
+	{
+		shouldclose = true;
+	}
+
 	if ( shouldclose )
 	{
 		ErrorHandler_Connect(buffer);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
index d601383..59494a0 100644
--- a/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
+++ b/src/backend/resourcemanager/communication/rmcomm_QD2RM.c
@@ -1440,6 +1440,7 @@ void *generateResourceRefreshHeartBeat(void *arg)
 				int sockres = 0;
 				while(true)
 				{
+					int on;
 					sockres = connect(fd,
 									  (struct sockaddr *)&server_addr,
 									  sizeof(server_addr));
@@ -1455,8 +1456,34 @@ void *generateResourceRefreshHeartBeat(void *arg)
 									  "failed to connect to resource manager, "
 									  "fd %d (errno %d)", fd, errno);
 							close(fd);
+							fd = -1;
 						}
 					}
+#ifdef	TCP_NODELAY
+					on = 1;
+					if (sockres == 0 &&
+						setsockopt(fd,
+								   IPPROTO_TCP, TCP_NODELAY,
+								   (char *) &on, sizeof(on)) < 0)
+					{
+						write_log("ERROR setsockopt(TCP_NODELAY) failed: %m");
+						close(fd);
+						fd = -1;
+						sockres = -1;
+					}
+#endif
+					on = 1;
+					if (sockres == 0 &&
+						setsockopt(fd,
+								   SOL_SOCKET, SO_KEEPALIVE,
+								   (char *) &on, sizeof(on)) < 0)
+					{
+						write_log("ERROR setsockopt(SO_KEEPALIVE) failed: %m");
+						close(fd);
+						fd = -1;
+						sockres = -1;
+					}
+
 					break;
 				}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/include/utils/network_utils.h
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/include/utils/network_utils.h b/src/backend/resourcemanager/include/utils/network_utils.h
index 4aa2cf0..a048131 100644
--- a/src/backend/resourcemanager/include/utils/network_utils.h
+++ b/src/backend/resourcemanager/include/utils/network_utils.h
@@ -109,6 +109,7 @@ int setConnectionNonBlocked(int fd);
 
 #define DRM_SOCKET_CONN_RETRY 5
 int  connectToServerRemote(const char *address,uint16_t port,int *clientfd);
+int setConnectionLongTermNoDelay(int fd);
 void closeConnectionRemote(int *clientfd);
 void returnAliveConnectionRemote(int 			*clientfd,
 								 const char 	*hostname,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e82ac3f1/src/backend/resourcemanager/utils/network_utils.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/utils/network_utils.c b/src/backend/resourcemanager/utils/network_utils.c
index b8bbab9..fbfe0a9 100644
--- a/src/backend/resourcemanager/utils/network_utils.c
+++ b/src/backend/resourcemanager/utils/network_utils.c
@@ -715,3 +715,23 @@ static void cleanupSocketConnectionPool(int code, Datum arg)
 	freePAIRRefList(&ResolvedHostnames, &addrlist);
 	cleanHASHTABLE(&ResolvedHostnames);
 }
+
+int setConnectionLongTermNoDelay(int fd)
+{
+	int on;
+#ifdef	TCP_NODELAY
+	on = 1;
+	if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &on, sizeof(on)) < 0)
+	{
+		elog(WARNING, "setsockopt(TCP_NODELAY) failed: %m");
+		return SYSTEM_CALL_ERROR;
+	}
+#endif
+	on = 1;
+	if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (char *) &on, sizeof(on)) < 0)
+	{
+		elog(WARNING, "setsockopt(SO_KEEPALIVE) failed: %m");
+		return SYSTEM_CALL_ERROR;
+	}
+	return FUNC_RETURN_OK;
+}