You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/07/18 02:19:44 UTC
[11/50] [abbrv] carbondata git commit:
[CARBONDATA-2655][BloomDataMap] BloomFilter datamap support in operator
[CARBONDATA-2655][BloomDataMap] BloomFilter datamap support in operator
Now queries with in expression on bloom index column can leverage the
BloomFilter datamap.
This closes #2445
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/202d099d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/202d099d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/202d099d
Branch: refs/heads/carbonstore
Commit: 202d099d631571a96acdf781749561ad3f0da36a
Parents: f911403
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sat Jul 7 22:19:53 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Thu Jul 12 16:38:51 2018 +0800
----------------------------------------------------------------------
.../datamap/bloom/BloomCoarseGrainDataMap.java | 48 ++++++++++++++++++--
.../bloom/BloomCoarseGrainDataMapFactory.java | 1 +
.../bloom/BloomCoarseGrainDataMapSuite.scala | 36 +++++++++++++++
3 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index 01bd804..96f3495 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -55,6 +55,8 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.InExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -178,6 +180,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
private List<BloomQueryModel> createQueryModel(Expression expression)
throws DictionaryGenerationException, UnsupportedEncodingException {
List<BloomQueryModel> queryModels = new ArrayList<BloomQueryModel>();
+ // bloomdatamap only support equalTo and In operators now
if (expression instanceof EqualToExpression) {
Expression left = ((EqualToExpression) expression).getLeft();
Expression right = ((EqualToExpression) expression).getRight();
@@ -186,7 +189,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
column = ((ColumnExpression) left).getColumnName();
if (this.name2Col.containsKey(column)) {
BloomQueryModel bloomQueryModel =
- buildQueryModelFromExpression((ColumnExpression) left, (LiteralExpression) right);
+ buildQueryModelForEqual((ColumnExpression) left, (LiteralExpression) right);
queryModels.add(bloomQueryModel);
}
return queryModels;
@@ -194,10 +197,35 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
column = ((ColumnExpression) right).getColumnName();
if (this.name2Col.containsKey(column)) {
BloomQueryModel bloomQueryModel =
- buildQueryModelFromExpression((ColumnExpression) right, (LiteralExpression) left);
+ buildQueryModelForEqual((ColumnExpression) right, (LiteralExpression) left);
queryModels.add(bloomQueryModel);
}
return queryModels;
+ } else {
+ LOGGER.warn("BloomFilter can only support the 'equal' filter like 'Col = PlainValue'");
+ }
+ } else if (expression instanceof InExpression) {
+ Expression left = ((InExpression) expression).getLeft();
+ Expression right = ((InExpression) expression).getRight();
+ String column;
+ if (left instanceof ColumnExpression && right instanceof ListExpression) {
+ column = ((ColumnExpression) left).getColumnName();
+ if (this.name2Col.containsKey(column)) {
+ List<BloomQueryModel> models =
+ buildQueryModelForIn((ColumnExpression) left, (ListExpression) right);
+ queryModels.addAll(models);
+ }
+ return queryModels;
+ } else if (left instanceof ListExpression && right instanceof ColumnExpression) {
+ column = ((ColumnExpression) right).getColumnName();
+ if (this.name2Col.containsKey(column)) {
+ List<BloomQueryModel> models =
+ buildQueryModelForIn((ColumnExpression) right, (ListExpression) left);
+ queryModels.addAll(models);
+ }
+ return queryModels;
+ } else {
+ LOGGER.warn("BloomFilter can only support the 'in' filter like 'Col in (PlainValues)'");
}
}
@@ -207,7 +235,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
return queryModels;
}
- private BloomQueryModel buildQueryModelFromExpression(ColumnExpression ce,
+ private BloomQueryModel buildQueryModelForEqual(ColumnExpression ce,
LiteralExpression le) throws DictionaryGenerationException, UnsupportedEncodingException {
String columnName = ce.getColumnName();
DataType dataType = ce.getDataType();
@@ -234,6 +262,20 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
return buildQueryModelInternal(this.name2Col.get(columnName), literalValue, dataType);
}
+ /**
+ * for `in` expressions, we use `equal` to handle it.
+ * Note that `in` operator needs at least one match not exactly match. since while doing pruning,
+ * we collect all the blocklets that will match the querymodel, this will not be a problem.
+ */
+ private List<BloomQueryModel> buildQueryModelForIn(ColumnExpression ce, ListExpression le)
+ throws DictionaryGenerationException, UnsupportedEncodingException {
+ List<BloomQueryModel> queryModels = new ArrayList<>();
+ for (Expression child : le.getChildren()) {
+ queryModels.add(buildQueryModelForEqual(ce, (LiteralExpression) child));
+ }
+ return queryModels;
+ }
+
private BloomQueryModel buildQueryModelInternal(CarbonColumn carbonColumn,
Object filterLiteralValue, DataType filterValueDataType) throws
DictionaryGenerationException, UnsupportedEncodingException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index bf56043..68cf45c 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -112,6 +112,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>();
// todo: support more optimize operations
optimizedOperations.add(ExpressionType.EQUALS);
+ optimizedOperations.add(ExpressionType.IN);
this.dataMapMeta = new DataMapMeta(this.dataMapName, indexedColumns, optimizedOperations);
LOGGER.info(String.format("DataMap %s works for %s with bloom size %d",
this.dataMapName, this.dataMapMeta, this.bloomFilterSize));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 14f8966..4dc1837 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -57,6 +57,9 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
}
private def checkQuery(dataMapName: String, shouldHit: Boolean = true) = {
+ /**
+ * queries that use equal operator
+ */
checkAnswer(
checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 1", dataMapName, shouldHit),
sql(s"select * from $normalTable where id = 1"))
@@ -85,6 +88,39 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
checkAnswer(
checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_999' and name='n1'", dataMapName, shouldHit),
sql(s"select * from $normalTable where city = 'city_999' and name='n1'"))
+
+ /**
+ * queries that use in operator
+ */
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (1)", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id in (1)"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (999)", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id in (999)"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in( 'city_1')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city in( 'city_1')"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city in ('city_999')"))
+ // query with two index_columns
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (1) and city in ('city_1')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id in (1) and city in ('city_1')"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (999) and city in ('city_999')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where id in (999) and city in ('city_999')"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_1') and id in (0)", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city in ('city_1') and id in (0)"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999') and name in ('n999')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city in ('city_999') and name in ('n999')"))
+ checkAnswer(
+ checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999') and name in ('n1')", dataMapName, shouldHit),
+ sql(s"select * from $normalTable where city in ('city_999') and name in ('n1')"))
+
checkAnswer(
sql(s"select min(id), max(id), min(name), max(name), min(city), max(city)" +
s" from $bloomDMSampleTable"),