You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2019/08/15 10:30:59 UTC
[hawq] 01/01: HAWQ-1748. Idle executors keep exist instead of quit
even when gp_vmem_idle_resource_timeout specified timer expires
This is an automated email from the ASF dual-hosted git repository.
huor pushed a commit to branch huor
in repository https://gitbox.apache.org/repos/asf/hawq.git
commit cfec1150eba4ba7d757cfea3fd7e884b765505b4
Author: Ruilong Huo <hu...@163.com>
AuthorDate: Thu Aug 15 18:21:11 2019 +0800
HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout specified timer expires
---
depends/dbcommon/Makefile.global | 8 ++++----
depends/storage/Makefile.global | 8 ++++----
depends/univplan/Makefile.global | 8 ++++----
src/backend/cdb/executormgr.c | 24 +++++++++++++++++++++++
src/backend/cdb/poolmgr.c | 42 ++++++++++++++++++++++++++++++++++++++++
src/backend/storage/lmgr/proc.c | 4 ++--
src/backend/tcop/postgres.c | 10 +++++++++-
src/backend/utils/misc/guc.c | 2 +-
src/include/cdb/executormgr.h | 3 +++
src/include/cdb/poolmgr.h | 2 ++
10 files changed, 95 insertions(+), 16 deletions(-)
diff --git a/depends/dbcommon/Makefile.global b/depends/dbcommon/Makefile.global
index e471d53..9ebc256 100644
--- a/depends/dbcommon/Makefile.global
+++ b/depends/dbcommon/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_dbcommon = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/depends/storage/Makefile.global b/depends/storage/Makefile.global
index 8fd25bf..4be6114 100644
--- a/depends/storage/Makefile.global
+++ b/depends/storage/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_storage = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/depends/univplan/Makefile.global b/depends/univplan/Makefile.global
index a291648..31c56ce 100644
--- a/depends/univplan/Makefile.global
+++ b/depends/univplan/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_univplan = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 5a1c15d..62473d7 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -1096,3 +1096,27 @@ executormgr_destory(SegmentDatabaseDescriptor *desc)
pfree(desc);
}
+bool executormgr_has_cached_executor()
+{
+ return executor_cache.cached_num > 0;
+}
+
+bool executormgr_clean_cached_executor_filter(PoolItem item)
+{
+ SegmentDatabaseDescriptor *desc = (SegmentDatabaseDescriptor *)item;
+ return desc->conn->asyncStatus == PGASYNC_IDLE;
+}
+
+void executormgr_clean_cached_executor()
+{
+ /* go through each cached executor */
+ int cleaned = 0;
+ if (!executor_cache.init)
+ {
+ return;
+ }
+
+ cleaned = poolmgr_clean(executor_cache.pool, (PoolMgrIterateFilter) executormgr_clean_cached_executor_filter);
+ elog(DEBUG5, "cleaned %d idle executors", cleaned);
+}
+
diff --git a/src/backend/cdb/poolmgr.c b/src/backend/cdb/poolmgr.c
index eae0f04..f7194f6 100644
--- a/src/backend/cdb/poolmgr.c
+++ b/src/backend/cdb/poolmgr.c
@@ -184,3 +184,45 @@ poolmgr_iterate(PoolMgrState *pool, PoolMgrIterateFilter filter, PoolMgrIterateC
return;
}
+int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter)
+{
+ int res = 0;
+ HASH_SEQ_STATUS hash_seq;
+ PoolItemEntry *entry;
+
+ if (!pool)
+ return res;
+
+ hash_seq_init(&hash_seq, pool->hashtable);
+ while ((entry = hash_seq_search(&hash_seq)))
+ {
+ ListCell *curr = list_head(entry->list);
+ ListCell *prev = NULL;
+ while (curr != NULL)
+ {
+ PoolItem item = lfirst(curr);
+ if (filter && !filter(item))
+ {
+ /* try next */
+ prev = curr;
+ curr = lnext(prev);
+ continue;
+ }
+
+ /* clean now */
+ res++;
+ pool->callback(item);
+ entry->list = list_delete_cell(entry->list, curr, prev);
+ if (prev != NULL)
+ {
+ curr = lnext(prev);
+ }
+ else
+ {
+ curr = list_head(entry->list);
+ }
+ }
+ }
+ return res;
+}
+
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 2d3097f..4fef10c 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -1399,9 +1399,9 @@ HandleClientWaitTimeout(void)
/*
* Free gangs to free up resources on the segDBs.
*/
- if (gangsExist())
+ if (executormgr_has_cached_executor())
{
- cleanupAllIdleGangs();
+ executormgr_clean_cached_executor();
}
}
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index cf0123e..c122280 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -4804,9 +4804,17 @@ PostgresMain(int argc, char *argv[], const char *username)
* This means giving the end user enough time to type in the next SQL statement
*
*/
- if (IdleSessionGangTimeout > 0 && gangsExist())
+ if (IdleSessionGangTimeout > 0 && executormgr_has_cached_executor())
+ {
if (!enable_sig_alarm( IdleSessionGangTimeout /* ms */, false))
+ {
elog(FATAL, "could not set timer for client wait timeout");
+ }
+ }
+ else if (IdleSessionGangTimeout == 0)
+ {
+ executormgr_clean_cached_executor();
+ }
}
IdleTracker_DeactivateProcess();
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0cf51df..b90f1ec 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5067,7 +5067,7 @@ static struct config_int ConfigureNamesInt[] =
{
{"gp_vmem_idle_resource_timeout", PGC_USERSET, CLIENT_CONN_OTHER,
gettext_noop("Sets the time a session can be idle (in milliseconds) before we release gangs on the segment DBs to free resources."),
- gettext_noop("A value of 0 turns off the timeout."),
+ gettext_noop("A value of 0 closes idle gangs at once."),
GUC_UNIT_MS | GUC_GPDB_ADDOPT
},
&IdleSessionGangTimeout,
diff --git a/src/include/cdb/executormgr.h b/src/include/cdb/executormgr.h
index 31d44fa..ce4442d 100644
--- a/src/include/cdb/executormgr.h
+++ b/src/include/cdb/executormgr.h
@@ -98,5 +98,8 @@ extern bool executormgr_connect(struct SegmentDatabaseDescriptor *desc,
bool is_writer, bool is_superuser);
extern void executormgr_free_executor(struct SegmentDatabaseDescriptor *desc);
+extern bool executormgr_has_cached_executor();
+extern void executormgr_clean_cached_executor();
+
#endif /* EXECUTORMGR_H */
diff --git a/src/include/cdb/poolmgr.h b/src/include/cdb/poolmgr.h
index 2821dd9..8408c25 100644
--- a/src/include/cdb/poolmgr.h
+++ b/src/include/cdb/poolmgr.h
@@ -32,6 +32,8 @@ typedef bool (*PoolMgrIterateFilter) (PoolItem item);
extern struct PoolMgrState *poolmgr_create_pool(MemoryContext ctx, PoolMgrCleanCallback callback);
extern bool poolmgr_drop_pool(struct PoolMgrState *pool);
+extern int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter);
+
extern PoolItem poolmgr_get_item_by_name(struct PoolMgrState *pool, const char *name);
extern PoolItem poolmgr_get_random_item(struct PoolMgrState *pool);
extern void poolmgr_put_item(struct PoolMgrState *pool, const char *name, PoolItem item);