You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2019/08/15 10:30:58 UTC
[hawq] branch huor updated (4b14aa1 -> cfec115)
This is an automated email from the ASF dual-hosted git repository.
huor pushed a change to branch huor
in repository https://gitbox.apache.org/repos/asf/hawq.git.
discard 4b14aa1 HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout specified timer expires
add d155b5b HAWQ-1747. Change pip downloading
new cfec115 HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout specified timer expires
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4b14aa1)
\
N -- N -- N refs/heads/huor (cfec115)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[hawq] 01/01: HAWQ-1748. Idle executors keep exist instead of quit
even when gp_vmem_idle_resource_timeout specified timer expires
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huor pushed a commit to branch huor
in repository https://gitbox.apache.org/repos/asf/hawq.git
commit cfec1150eba4ba7d757cfea3fd7e884b765505b4
Author: Ruilong Huo <hu...@163.com>
AuthorDate: Thu Aug 15 18:21:11 2019 +0800
HAWQ-1748. Idle executors keep exist instead of quit even when gp_vmem_idle_resource_timeout specified timer expires
---
depends/dbcommon/Makefile.global | 8 ++++----
depends/storage/Makefile.global | 8 ++++----
depends/univplan/Makefile.global | 8 ++++----
src/backend/cdb/executormgr.c | 24 +++++++++++++++++++++++
src/backend/cdb/poolmgr.c | 42 ++++++++++++++++++++++++++++++++++++++++
src/backend/storage/lmgr/proc.c | 4 ++--
src/backend/tcop/postgres.c | 10 +++++++++-
src/backend/utils/misc/guc.c | 2 +-
src/include/cdb/executormgr.h | 3 +++
src/include/cdb/poolmgr.h | 2 ++
10 files changed, 95 insertions(+), 16 deletions(-)
diff --git a/depends/dbcommon/Makefile.global b/depends/dbcommon/Makefile.global
index e471d53..9ebc256 100644
--- a/depends/dbcommon/Makefile.global
+++ b/depends/dbcommon/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_dbcommon = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/depends/storage/Makefile.global b/depends/storage/Makefile.global
index 8fd25bf..4be6114 100644
--- a/depends/storage/Makefile.global
+++ b/depends/storage/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_storage = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/depends/univplan/Makefile.global b/depends/univplan/Makefile.global
index a291648..31c56ce 100644
--- a/depends/univplan/Makefile.global
+++ b/depends/univplan/Makefile.global
@@ -20,15 +20,15 @@
# A makefile that integrate building this module with hawq
#------------------------------------------------------------------------------
-prefix := /usr/local/hawq
-enable_debug = no
+prefix := /data/workspace/hawq
+enable_debug = yes
enable_coverage = no
with_univplan = yes
# Support for VPATH builds
vpath_build = no
-abs_top_srcdir = /data/apache-hawq
-abs_top_builddir = /data/apache-hawq
+abs_top_srcdir = /data/repository/apache-hawq
+abs_top_builddir = /data/repository/apache-hawq
ifneq ($(vpath_build),yes)
top_srcdir = $(top_builddir)
diff --git a/src/backend/cdb/executormgr.c b/src/backend/cdb/executormgr.c
index 5a1c15d..62473d7 100644
--- a/src/backend/cdb/executormgr.c
+++ b/src/backend/cdb/executormgr.c
@@ -1096,3 +1096,27 @@ executormgr_destory(SegmentDatabaseDescriptor *desc)
pfree(desc);
}
+bool executormgr_has_cached_executor()
+{
+ return executor_cache.cached_num > 0;
+}
+
+bool executormgr_clean_cached_executor_filter(PoolItem item)
+{
+ SegmentDatabaseDescriptor *desc = (SegmentDatabaseDescriptor *)item;
+ return desc->conn->asyncStatus == PGASYNC_IDLE;
+}
+
+void executormgr_clean_cached_executor()
+{
+ /* go through each cached executor */
+ int cleaned = 0;
+ if (!executor_cache.init)
+ {
+ return;
+ }
+
+ cleaned = poolmgr_clean(executor_cache.pool, (PoolMgrIterateFilter) executormgr_clean_cached_executor_filter);
+ elog(DEBUG5, "cleaned %d idle executors", cleaned);
+}
+
diff --git a/src/backend/cdb/poolmgr.c b/src/backend/cdb/poolmgr.c
index eae0f04..f7194f6 100644
--- a/src/backend/cdb/poolmgr.c
+++ b/src/backend/cdb/poolmgr.c
@@ -184,3 +184,45 @@ poolmgr_iterate(PoolMgrState *pool, PoolMgrIterateFilter filter, PoolMgrIterateC
return;
}
+int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter)
+{
+ int res = 0;
+ HASH_SEQ_STATUS hash_seq;
+ PoolItemEntry *entry;
+
+ if (!pool)
+ return res;
+
+ hash_seq_init(&hash_seq, pool->hashtable);
+ while ((entry = hash_seq_search(&hash_seq)))
+ {
+ ListCell *curr = list_head(entry->list);
+ ListCell *prev = NULL;
+ while (curr != NULL)
+ {
+ PoolItem item = lfirst(curr);
+ if (filter && !filter(item))
+ {
+ /* try next */
+ prev = curr;
+ curr = lnext(prev);
+ continue;
+ }
+
+ /* clean now */
+ res++;
+ pool->callback(item);
+ entry->list = list_delete_cell(entry->list, curr, prev);
+ if (prev != NULL)
+ {
+ curr = lnext(prev);
+ }
+ else
+ {
+ curr = list_head(entry->list);
+ }
+ }
+ }
+ return res;
+}
+
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 2d3097f..4fef10c 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -1399,9 +1399,9 @@ HandleClientWaitTimeout(void)
/*
* Free gangs to free up resources on the segDBs.
*/
- if (gangsExist())
+ if (executormgr_has_cached_executor())
{
- cleanupAllIdleGangs();
+ executormgr_clean_cached_executor();
}
}
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index cf0123e..c122280 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -4804,9 +4804,17 @@ PostgresMain(int argc, char *argv[], const char *username)
* This means giving the end user enough time to type in the next SQL statement
*
*/
- if (IdleSessionGangTimeout > 0 && gangsExist())
+ if (IdleSessionGangTimeout > 0 && executormgr_has_cached_executor())
+ {
if (!enable_sig_alarm( IdleSessionGangTimeout /* ms */, false))
+ {
elog(FATAL, "could not set timer for client wait timeout");
+ }
+ }
+ else if (IdleSessionGangTimeout == 0)
+ {
+ executormgr_clean_cached_executor();
+ }
}
IdleTracker_DeactivateProcess();
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0cf51df..b90f1ec 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -5067,7 +5067,7 @@ static struct config_int ConfigureNamesInt[] =
{
{"gp_vmem_idle_resource_timeout", PGC_USERSET, CLIENT_CONN_OTHER,
gettext_noop("Sets the time a session can be idle (in milliseconds) before we release gangs on the segment DBs to free resources."),
- gettext_noop("A value of 0 turns off the timeout."),
+ gettext_noop("A value of 0 closes idle gangs at once."),
GUC_UNIT_MS | GUC_GPDB_ADDOPT
},
&IdleSessionGangTimeout,
diff --git a/src/include/cdb/executormgr.h b/src/include/cdb/executormgr.h
index 31d44fa..ce4442d 100644
--- a/src/include/cdb/executormgr.h
+++ b/src/include/cdb/executormgr.h
@@ -98,5 +98,8 @@ extern bool executormgr_connect(struct SegmentDatabaseDescriptor *desc,
bool is_writer, bool is_superuser);
extern void executormgr_free_executor(struct SegmentDatabaseDescriptor *desc);
+extern bool executormgr_has_cached_executor();
+extern void executormgr_clean_cached_executor();
+
#endif /* EXECUTORMGR_H */
diff --git a/src/include/cdb/poolmgr.h b/src/include/cdb/poolmgr.h
index 2821dd9..8408c25 100644
--- a/src/include/cdb/poolmgr.h
+++ b/src/include/cdb/poolmgr.h
@@ -32,6 +32,8 @@ typedef bool (*PoolMgrIterateFilter) (PoolItem item);
extern struct PoolMgrState *poolmgr_create_pool(MemoryContext ctx, PoolMgrCleanCallback callback);
extern bool poolmgr_drop_pool(struct PoolMgrState *pool);
+extern int poolmgr_clean(struct PoolMgrState *pool, PoolMgrIterateFilter filter);
+
extern PoolItem poolmgr_get_item_by_name(struct PoolMgrState *pool, const char *name);
extern PoolItem poolmgr_get_random_item(struct PoolMgrState *pool);
extern void poolmgr_put_item(struct PoolMgrState *pool, const char *name, PoolItem item);