You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2016/01/06 08:19:32 UTC
incubator-hawq git commit: HAWQ-300. Fix out of memory during loading
large volume data on large cluster with YARN mode
Repository: incubator-hawq
Updated Branches:
refs/heads/master 1e535bd4d -> 7e0ff4193
HAWQ-300. Fix out of memory during loading large volume data on large cluster with YARN mode
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7e0ff419
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7e0ff419
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7e0ff419
Branch: refs/heads/master
Commit: 7e0ff41933e4efc58918656dfdedae496432a3f0
Parents: 1e535bd
Author: Ruilong Huo <rh...@pivotal.io>
Authored: Tue Jan 5 22:15:54 2016 -0800
Committer: Ruilong Huo <rh...@pivotal.io>
Committed: Tue Jan 5 22:15:54 2016 -0800
----------------------------------------------------------------------
src/backend/executor/execMain.c | 18 +++++++++++++++++-
src/backend/utils/init/globals.c | 1 +
src/backend/utils/misc/guc.c | 10 ++++++++++
src/include/miscadmin.h | 1 +
src/test/regress/expected/parquet_compression.out | 2 +-
src/test/regress/sql/parquet_compression.sql | 2 +-
6 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/executor/execMain.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index f5a2038..831e557 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -109,6 +109,8 @@
#include "cdb/cdbquerycontextdispatching.h"
#include "optimizer/prep.h"
+#include "resourcemanager/dynrm.h"
+
extern bool filesystem_support_truncate;
typedef struct evalPlanQual
@@ -440,8 +442,22 @@ ExecutorStart(QueryDesc *queryDesc, int eflags)
* resource queues.
*/
if (queryDesc->plannedstmt->query_mem > 0) {
+
+ uint64 memAvailableBytes = 0;
+
+ /* With resouce manager in NONE mode, we assign memory quota for operators normally */
+ if ( strcasecmp(rm_global_rm_type, HAWQDRM_CONFFILE_SVRTYPE_VAL_NONE) == 0 )
+ {
+ memAvailableBytes = queryDesc->plannedstmt->query_mem;
+ }
+ /* With resouce manager in YARN/MESOS mode, we assign memory quota for operators conservatively */
+ else
+ {
+ memAvailableBytes = queryDesc->plannedstmt->query_mem * hawq_re_memory_quota_allocation_ratio;
+ }
+
PolicyEagerFreeAssignOperatorMemoryKB(queryDesc->plannedstmt,
- queryDesc->plannedstmt->query_mem);
+ memAvailableBytes);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/utils/init/globals.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c
index 3ea1428..951b23d 100644
--- a/src/backend/utils/init/globals.c
+++ b/src/backend/utils/init/globals.c
@@ -150,4 +150,5 @@ int hawq_re_memory_overcommit_max = 8192;
#else
int hawq_re_memory_overcommit_max = 8192;
#endif
+double hawq_re_memory_quota_allocation_ratio = 0.5;
int gp_vmem_protect_gang_cache_limit = 500;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index becc9c9..3f591d4 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6956,6 +6956,16 @@ static struct config_real ConfigureNamesReal[] =
3.0, 1.0, 100.0, NULL, NULL
},
+ {
+ {"hawq_re_memory_quota_allocation_ratio", PGC_USERSET, RESOURCES_MEM,
+ gettext_noop("Sets the ratio for memory quota allocation during query optimization."),
+ NULL,
+ GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE
+ },
+ &hawq_re_memory_quota_allocation_ratio,
+ 0.5, 0.0, 1.0, NULL, NULL
+ },
+
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/include/miscadmin.h
----------------------------------------------------------------------
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 8a90b5e..ce362e4 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -314,6 +314,7 @@ extern int VacuumCostBalance;
extern bool VacuumCostActive;
extern int hawq_re_memory_overcommit_max;
+extern double hawq_re_memory_quota_allocation_ratio;
extern int gp_vmem_protect_gang_cache_limit;
/* in tcop/postgres.c */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/test/regress/expected/parquet_compression.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/parquet_compression.out b/src/test/regress/expected/parquet_compression.out
index 5c01573..34feee3 100644
--- a/src/test/regress/expected/parquet_compression.out
+++ b/src/test/regress/expected/parquet_compression.out
@@ -26,7 +26,7 @@ drop table parquet_snappy_part_unc;
ERROR: table "parquet_snappy_part_unc" does not exist
drop table parquet_gzip_2;
ERROR: table "parquet_gzip_2" does not exist
-alter resource queue pg_default with ( vseg_resource_quota='mem:2gb');
+alter resource queue pg_default with ( vseg_resource_quota='mem:4gb');
--set statement_mem='1999MB';
--end_ignore
--Datatypes covered: text,bytea,varchar,bit varying
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/test/regress/sql/parquet_compression.sql
----------------------------------------------------------------------
diff --git a/src/test/regress/sql/parquet_compression.sql b/src/test/regress/sql/parquet_compression.sql
index 7a61d33..6e6bf3d 100644
--- a/src/test/regress/sql/parquet_compression.sql
+++ b/src/test/regress/sql/parquet_compression.sql
@@ -19,7 +19,7 @@ drop table parquet_snappy_part;
drop table parquet_snappy_part_unc;
drop table parquet_gzip_2;
-alter resource queue pg_default with ( vseg_resource_quota='mem:2gb');
+alter resource queue pg_default with ( vseg_resource_quota='mem:4gb');
--set statement_mem='1999MB';
--end_ignore