You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2018/05/15 03:20:45 UTC
incubator-hawq git commit: HAWQ-1608. Implement Printing Runtime
Filter Information For "explain analyze". Change GUC
hawq_hashjoin_bloomfilter to bool.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 755f8ae6d -> 25ed0ae26
HAWQ-1608. Implement Printing Runtime Filter Information For "explain analyze". Change GUC hawq_hashjoin_bloomfilter to bool.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/25ed0ae2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/25ed0ae2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/25ed0ae2
Branch: refs/heads/master
Commit: 25ed0ae26028a2d3f784fd75ad9db1d3f888539f
Parents: 755f8ae
Author: Wen Lin <wl...@pivotal.io>
Authored: Mon May 14 17:39:38 2018 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Mon May 14 17:39:38 2018 +0800
----------------------------------------------------------------------
src/backend/cdb/cdbvars.c | 4 +-
src/backend/executor/execParquetScan.c | 6 +++
src/backend/executor/nodeHash.c | 14 ++++++
src/backend/executor/nodeTableScan.c | 2 +-
src/backend/utils/misc/guc.c | 19 ++++----
src/include/cdb/cdbvars.h | 2 +-
.../feature/query/test_hashjoin_bloomfilter.cpp | 51 ++++++++++++++++++++
7 files changed, 84 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index c017fad..3fe13be 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -233,8 +233,8 @@ int gp_hashjoin_metadata_memory_percent = 20;
int gp_hashagg_spillbatch_min = 0;
int gp_hashagg_spillbatch_max = 0;
-/* hash join to use bloom filter: default to 0, means not used */
-int hawq_hashjoin_bloomfilter = 0;
+/* hash join to use bloom filter: default is false, means not used */
+bool hawq_hashjoin_bloomfilter = false;
/* maximum memory size for one Bloom filter */
char* hawq_hashjoin_bloomfilter_max_memory_size;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/execParquetScan.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execParquetScan.c b/src/backend/executor/execParquetScan.c
index 13b93eb..3ad8700 100644
--- a/src/backend/executor/execParquetScan.c
+++ b/src/backend/executor/execParquetScan.c
@@ -87,6 +87,12 @@ ParquetScanNext(ScanState *scanState)
Assert(node->opaque != NULL &&
node->opaque->scandesc != NULL);
+ /* push down Bloom filter */
+ if (node->opaque->scandesc->rfState == NULL &&
+ scanState->runtimeFilter != NULL)
+ {
+ node->opaque->scandesc->rfState = scanState->runtimeFilter;
+ }
parquet_getnext(node->opaque->scandesc, node->ss.ps.state->es_direction, node->ss.ss_ScanTupleSlot);
return node->ss.ss_ScanTupleSlot;
}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/nodeHash.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c
index ec31d9a..96a31e7 100644
--- a/src/backend/executor/nodeHash.c
+++ b/src/backend/executor/nodeHash.c
@@ -1480,6 +1480,20 @@ ExecHashTableExplainEnd(PlanState *planstate, struct StringInfoData *buf)
hashtable->nbatch - stats->nonemptybatches);
appendStringInfoChar(buf, '\n');
}
+
+ /* Report Bloom filter statistics. */
+ if (hashtable->bloomfilter != NULL)
+ {
+ BloomFilter bf = hashtable->bloomfilter;
+ appendStringInfo(buf,"Bloom filter, inner table row number:%d, "
+ "outer table checked row number:%d, "
+ "outer table matched row number:%d, "
+ "outer table filtered row number:%d, filtered rate:%.3f",
+ bf->nInserted, bf->nTested, bf->nMatched, bf->nTested - bf->nMatched,
+ bf->nTested == 0 ? 0 : (float)((float)(bf->nTested - bf->nMatched)/(float)(bf->nTested)));
+ appendStringInfoChar(buf, '\n');
+ }
+
} /* ExecHashTableExplainEnd */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/nodeTableScan.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/nodeTableScan.c b/src/backend/executor/nodeTableScan.c
index b067417..03d4d3d 100644
--- a/src/backend/executor/nodeTableScan.c
+++ b/src/backend/executor/nodeTableScan.c
@@ -79,7 +79,7 @@ FreeScanRuntimefilterState(RuntimeFilterState* rfstate)
if (bf != NULL && bf->isCreated)
{
elog(DEBUG3, "Hash Join, inner table row number:%d, "
- "outer table total row number:%d, "
+ "outer table checked row number:%d, "
"outer table matched row number:%d, "
"outer table filtered row number:%d, filtered rate:%.3f",
bf->nInserted, bf->nTested, bf->nMatched, bf->nTested - bf->nMatched,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index d59e34f..6577b4c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4421,6 +4421,15 @@ static struct config_bool ConfigureNamesBool[] =
false, NULL, NULL
},
+ {
+ {"hawq_hashjoin_bloomfilter", PGC_USERSET, GP_ARRAY_TUNING,
+ gettext_noop("Use bloomfilter in hash join"),
+ gettext_noop("Use bloomfilter may speed up hash join performance"),
+ GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL | GUC_GPDB_ADDOPT
+ },
+ &hawq_hashjoin_bloomfilter,
+ false, NULL, NULL
+ },
/* End-of-list marker */
{
@@ -5836,16 +5845,6 @@ static struct config_int ConfigureNamesInt[] =
},
{
- {"hawq_hashjoin_bloomfilter", PGC_USERSET, GP_ARRAY_TUNING,
- gettext_noop("Use bloomfilter in hash join"),
- gettext_noop("Use bloomfilter may speed up hash join performance"),
- GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL | GUC_GPDB_ADDOPT
- },
- &hawq_hashjoin_bloomfilter,
- 0, 0, 1, NULL, NULL
- },
-
- {
{"gp_motion_slice_noop", PGC_USERSET, GP_ARRAY_TUNING,
gettext_noop("Make motion nodes in certain slices noop"),
gettext_noop("Make motion nodes noop, to help analyze performace"),
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/include/cdb/cdbvars.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index 517ed67..e278b22 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -937,7 +937,7 @@ extern int gp_hashagg_spillbatch_min;
extern int gp_hashagg_spillbatch_max;
/* Hashjoin use bloom filter */
-extern int hawq_hashjoin_bloomfilter;
+extern bool hawq_hashjoin_bloomfilter;
/* Maximum memory size for one Bloom filter */
extern char* hawq_hashjoin_bloomfilter_max_memory_size;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/test/feature/query/test_hashjoin_bloomfilter.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/query/test_hashjoin_bloomfilter.cpp b/src/test/feature/query/test_hashjoin_bloomfilter.cpp
new file mode 100644
index 0000000..2adb24b
--- /dev/null
+++ b/src/test/feature/query/test_hashjoin_bloomfilter.cpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "gtest/gtest.h"
+#include "lib/command.h"
+#include "lib/hawq_config.h"
+#include "lib/sql_util.h"
+
+using hawq::test::SQLUtility;
+using std::string;
+using hawq::test::Command;
+
+class TestHashJoinBloomFilter: public ::testing::Test
+{
+ public:
+ TestHashJoinBloomFilter() {}
+ ~TestHashJoinBloomFilter() {}
+};
+
+TEST_F(TestHashJoinBloomFilter, BasicTest)
+{
+ SQLUtility util;
+ hawq::test::HawqConfig hawq_config;
+ util.execute("drop table if exists fact;");
+ util.execute("create table fact(c1 int) WITH(appendonly=true, ORIENTATION=parquet) distributed by (c1);");
+ util.execute("insert into fact select generate_series(1, 300000);");
+ util.query("select * from fact;", 300000);
+ util.execute("drop table if exists dim;");
+ util.execute("create table dim(c1 int, c2 int) distributed by (c1) ;");
+ util.execute("insert into dim values(3,1),(1,2),(1,3),(1,1),(2,3),(2,5),(2,6),(1000,7),(2000,1),(3000,2)");
+ util.query("select * from dim;", 10);
+ util.query("select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4", 7);
+ util.query("set hawq_hashjoin_bloomfilter=true; select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4", 7);
+ util.execute("set hawq_hashjoin_bloomfilter=true; explain analyze select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4");
+ util.execute("drop table dim;");
+ util.execute("drop table fact;");
+}