You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2018/05/15 03:20:45 UTC

incubator-hawq git commit: HAWQ-1608. Implement Printing Runtime Filter Information For "explain analyze". Change GUC hawq_hashjoin_bloomfilter to bool.

Repository: incubator-hawq
Updated Branches:
  refs/heads/master 755f8ae6d -> 25ed0ae26


HAWQ-1608. Implement Printing Runtime Filter Information For "explain analyze". Change GUC hawq_hashjoin_bloomfilter to bool.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/25ed0ae2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/25ed0ae2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/25ed0ae2

Branch: refs/heads/master
Commit: 25ed0ae26028a2d3f784fd75ad9db1d3f888539f
Parents: 755f8ae
Author: Wen Lin <wl...@pivotal.io>
Authored: Mon May 14 17:39:38 2018 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Mon May 14 17:39:38 2018 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbvars.c                       |  4 +-
 src/backend/executor/execParquetScan.c          |  6 +++
 src/backend/executor/nodeHash.c                 | 14 ++++++
 src/backend/executor/nodeTableScan.c            |  2 +-
 src/backend/utils/misc/guc.c                    | 19 ++++----
 src/include/cdb/cdbvars.h                       |  2 +-
 .../feature/query/test_hashjoin_bloomfilter.cpp | 51 ++++++++++++++++++++
 7 files changed, 84 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index c017fad..3fe13be 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -233,8 +233,8 @@ int			gp_hashjoin_metadata_memory_percent = 20;
 int 		gp_hashagg_spillbatch_min = 0;
 int 		gp_hashagg_spillbatch_max = 0;
 
-/* hash join to use bloom filter: default to 0, means not used */
-int 	 	hawq_hashjoin_bloomfilter = 0;
+/* hash join to use bloom filter: default is false, means not used */
+bool 	 	hawq_hashjoin_bloomfilter = false;
 
 /* maximum memory size for one Bloom filter */
 char*		hawq_hashjoin_bloomfilter_max_memory_size;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/execParquetScan.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execParquetScan.c b/src/backend/executor/execParquetScan.c
index 13b93eb..3ad8700 100644
--- a/src/backend/executor/execParquetScan.c
+++ b/src/backend/executor/execParquetScan.c
@@ -87,6 +87,12 @@ ParquetScanNext(ScanState *scanState)
 	Assert(node->opaque != NULL &&
 		   node->opaque->scandesc != NULL);
 
+	/* push down Bloom filter */
+	if (node->opaque->scandesc->rfState == NULL &&
+			scanState->runtimeFilter != NULL)
+	{
+		node->opaque->scandesc->rfState = scanState->runtimeFilter;
+	}
 	parquet_getnext(node->opaque->scandesc, node->ss.ps.state->es_direction, node->ss.ss_ScanTupleSlot);
 	return node->ss.ss_ScanTupleSlot;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/nodeHash.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c
index ec31d9a..96a31e7 100644
--- a/src/backend/executor/nodeHash.c
+++ b/src/backend/executor/nodeHash.c
@@ -1480,6 +1480,20 @@ ExecHashTableExplainEnd(PlanState *planstate, struct StringInfoData *buf)
                              hashtable->nbatch - stats->nonemptybatches);
         appendStringInfoChar(buf, '\n');
     }
+
+    /* Report Bloom filter statistics. */
+    if (hashtable->bloomfilter != NULL)
+    {
+        BloomFilter bf = hashtable->bloomfilter;
+        appendStringInfo(buf,"Bloom filter, inner table row number:%d, "
+                        "outer table checked row number:%d, "
+                        "outer table matched row number:%d, "
+                        "outer table filtered row number:%d, filtered rate:%.3f",
+                        bf->nInserted, bf->nTested, bf->nMatched, bf->nTested - bf->nMatched,
+                        bf->nTested == 0 ? 0 : (float)((float)(bf->nTested - bf->nMatched)/(float)(bf->nTested)));
+        appendStringInfoChar(buf, '\n');
+    }
+
 }                               /* ExecHashTableExplainEnd */
 
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/executor/nodeTableScan.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/nodeTableScan.c b/src/backend/executor/nodeTableScan.c
index b067417..03d4d3d 100644
--- a/src/backend/executor/nodeTableScan.c
+++ b/src/backend/executor/nodeTableScan.c
@@ -79,7 +79,7 @@ FreeScanRuntimefilterState(RuntimeFilterState* rfstate)
 	if (bf != NULL && bf->isCreated)
 	{
 		elog(DEBUG3, "Hash Join, inner table row number:%d, "
-					 "outer table total row number:%d, "
+					 "outer table checked row number:%d, "
 					 "outer table matched row number:%d, "
 					 "outer table filtered row number:%d, filtered rate:%.3f",
 			 bf->nInserted, bf->nTested, bf->nMatched, bf->nTested - bf->nMatched,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index d59e34f..6577b4c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -4421,6 +4421,15 @@ static struct config_bool ConfigureNamesBool[] =
 		false, NULL, NULL
 	},
 
+	{
+		{"hawq_hashjoin_bloomfilter", PGC_USERSET, GP_ARRAY_TUNING,
+			gettext_noop("Use bloomfilter in hash join"),
+			gettext_noop("Use bloomfilter may speed up hash join performance"),
+			GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL | GUC_GPDB_ADDOPT
+		},
+		&hawq_hashjoin_bloomfilter,
+		false, NULL, NULL
+	},
 
 	/* End-of-list marker */
 	{
@@ -5836,16 +5845,6 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
-		{"hawq_hashjoin_bloomfilter", PGC_USERSET, GP_ARRAY_TUNING,
-		 gettext_noop("Use bloomfilter in hash join"),
-		 gettext_noop("Use bloomfilter may speed up hash join performance"),
-		 GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL | GUC_GPDB_ADDOPT
-		},
-		&hawq_hashjoin_bloomfilter,
-		0, 0, 1, NULL, NULL
-	},
-
-	{
 		{"gp_motion_slice_noop", PGC_USERSET, GP_ARRAY_TUNING,
 		 gettext_noop("Make motion nodes in certain slices noop"),
 		 gettext_noop("Make motion nodes noop, to help analyze performace"),

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/include/cdb/cdbvars.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index 517ed67..e278b22 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -937,7 +937,7 @@ extern int gp_hashagg_spillbatch_min;
 extern int gp_hashagg_spillbatch_max;
 
 /* Hashjoin use bloom filter */
-extern int hawq_hashjoin_bloomfilter;
+extern bool hawq_hashjoin_bloomfilter;
 
 /* Maximum memory size for one Bloom filter */
 extern char* hawq_hashjoin_bloomfilter_max_memory_size;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/25ed0ae2/src/test/feature/query/test_hashjoin_bloomfilter.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/query/test_hashjoin_bloomfilter.cpp b/src/test/feature/query/test_hashjoin_bloomfilter.cpp
new file mode 100644
index 0000000..2adb24b
--- /dev/null
+++ b/src/test/feature/query/test_hashjoin_bloomfilter.cpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "gtest/gtest.h"
+#include "lib/command.h"
+#include "lib/hawq_config.h"
+#include "lib/sql_util.h"
+
+using hawq::test::SQLUtility;
+using std::string;
+using hawq::test::Command;
+
+class TestHashJoinBloomFilter: public ::testing::Test
+{
+    public:
+        TestHashJoinBloomFilter() {}
+        ~TestHashJoinBloomFilter() {}
+};
+
+TEST_F(TestHashJoinBloomFilter, BasicTest)
+{
+    SQLUtility util;
+    hawq::test::HawqConfig hawq_config;
+    util.execute("drop table if exists fact;");
+    util.execute("create table fact(c1 int) WITH(appendonly=true, ORIENTATION=parquet) distributed by (c1);");
+    util.execute("insert into fact select generate_series(1, 300000);");
+    util.query("select * from fact;", 300000);
+    util.execute("drop table if exists dim;");
+    util.execute("create table dim(c1 int, c2 int)  distributed by (c1) ;");
+    util.execute("insert into dim values(3,1),(1,2),(1,3),(1,1),(2,3),(2,5),(2,6),(1000,7),(2000,1),(3000,2)");
+    util.query("select * from dim;", 10);
+    util.query("select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4", 7);
+    util.query("set hawq_hashjoin_bloomfilter=true; select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4", 7);
+    util.execute("set hawq_hashjoin_bloomfilter=true; explain analyze select * from fact, dim where fact.c1 = dim.c1 and dim.c2<4");
+    util.execute("drop table dim;");
+    util.execute("drop table fact;");
+}