You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/02/28 01:17:12 UTC
carbondata git commit: [CARBONDATA-2206] Fixed lucene datamap
evaluation issue in executor
Repository: carbondata
Updated Branches:
refs/heads/datamap 24ebfbbda -> b6880e148
[CARBONDATA-2206] Fixed lucene datamap evaluation issue in executor
In case of MatchExpression it should return same bitset from RowLevelFilterExecuterImpl
This closes #2010
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b6880e14
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b6880e14
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b6880e14
Branch: refs/heads/datamap
Commit: b6880e14888b23d5841181ce991c640a27ed646d
Parents: 24ebfbb
Author: ravipesala <ra...@gmail.com>
Authored: Tue Feb 27 19:14:15 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Feb 28 09:16:58 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapChooser.java | 3 +-
.../core/scan/expression/MatchExpression.java | 57 ++++++++++++++++++++
.../executer/RowLevelFilterExecuterImpl.java | 4 ++
.../lucene/LuceneFineGrainDataMapSuite.scala | 8 ++-
.../datamap/expression/MatchExpression.java | 56 -------------------
.../spark/sql/optimizer/CarbonFilters.scala | 3 +-
6 files changed, 67 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index c8c971d..f9214a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -228,7 +228,8 @@ public class DataMapChooser {
private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions,
Set<ExpressionType> expressionTypes) {
- if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH)) {
+ if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH) &&
+ expressionTypes.contains(ExpressionType.TEXT_MATCH)) {
// TODO: fix it with right logic
return true;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
new file mode 100644
index 0000000..3677b51
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+@InterfaceAudience.Internal
+public class MatchExpression extends Expression {
+ private String queryString;
+
+ public MatchExpression(String queryString) {
+ this.queryString = queryString;
+ }
+
+ @Override
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return new ExpressionResult(DataTypes.BOOLEAN,true);
+ }
+
+ @Override
+ public ExpressionType getFilterExpressionType() {
+ return ExpressionType.TEXT_MATCH;
+ }
+
+ @Override
+ public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+
+ }
+
+ @Override
+ public String getString() {
+ return queryString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index c18f090..44fd6b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -50,6 +50,7 @@ import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.MatchExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
@@ -190,6 +191,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ if (exp instanceof MatchExpression) {
+ return rawBlockletColumnChunks.getBitSetGroup();
+ }
readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 4766281..5e28e8a 100644
--- a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -61,12 +61,10 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- sql("SELECT * FROM datamap_test ORDER BY id").show
-
// sql("select * from normal_test where name='n34000'").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')").show
+ checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')"), sql(s"select * from datamap_test where name='n10'"))
+// checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')"), sql(s"SELECT * FROM datamap_test WHERE name like 'n10%'"))
+ checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')"), sql(s"SELECT * FROM datamap_test WHERE city='c020'"))
// checkAnswer(
// sql("select * from datamap_test where match('name:n34000')"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
deleted file mode 100644
index fceb729..0000000
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.expression;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.ExpressionResult;
-import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-
-@InterfaceAudience.Internal
-public class MatchExpression extends Expression {
- private String queryString;
-
- public MatchExpression(String queryString) {
- this.queryString = queryString;
- }
-
- @Override
- public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return null;
- }
-
- @Override
- public ExpressionType getFilterExpressionType() {
- return ExpressionType.TEXT_MATCH;
- }
-
- @Override
- public void findAndSetChild(Expression oldExpr, Expression newExpr) {
-
- }
-
- @Override
- public String getString() {
- return queryString;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b6880e14/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 936f521..4a399cf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.PartitionMapFileStore
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
@@ -41,7 +41,6 @@ import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.datamap.TextMatch
-import org.apache.carbondata.datamap.expression.MatchExpression
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil