You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/22 01:34:32 UTC
[31/50] [abbrv] carbondata git commit: [CARBONDATA-2614] Fix the
error when using FG in search mode and the prune result is none
[CARBONDATA-2614] Fix the error when using FG in search mode and the prune result is none
the prune result is none, and can not set datamapWritePath, which will not generate bitSegGroup in org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl#applyFilter(org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks, boolean), it mean the bitSetGroup is null. It will throw Fix the error when using FG in search mode and the prune result is none
This closes #2378
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5593d164
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5593d164
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5593d164
Branch: refs/heads/carbonstore
Commit: 5593d1646b075847a84ac2891fd25e78201451fe
Parents: 60dfdd3
Author: xubo245 <xu...@huawei.com>
Authored: Fri Jun 15 18:50:26 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Jun 18 20:29:59 2018 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/scan/model/QueryModel.java | 13 +++++++++++++
.../apache/carbondata/hadoop/CarbonRecordReader.java | 2 +-
.../testsuite/detailquery/SearchModeTestCase.scala | 1 +
.../carbondata/store/worker/SearchRequestHandler.java | 1 +
4 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index de11d11..55dafb9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -109,6 +109,11 @@ public class QueryModel {
*/
private boolean requiredRowId;
+ /**
+ * whether it is FG with search mode
+ */
+ private boolean isFG;
+
private QueryModel(CarbonTable carbonTable) {
tableBlockInfos = new ArrayList<TableBlockInfo>();
invalidSegmentIds = new ArrayList<>();
@@ -370,6 +375,14 @@ public class QueryModel {
this.requiredRowId = requiredRowId;
}
+ public boolean isFG() {
+ return isFG;
+ }
+
+ public void setFG(boolean FG) {
+ isFG = FG;
+ }
+
@Override
public String toString() {
return String.format("scan on table %s.%s, %d projection columns with filter (%s)",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 4911e41..6b56382 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -80,7 +80,7 @@ public class CarbonRecordReader<T> extends AbstractRecordReader<T> {
}
// It should use the exists tableBlockInfos if tableBlockInfos of queryModel is not empty
// otherwise the prune is no use before this method
- if (queryModel.getTableBlockInfos().isEmpty()) {
+ if (!queryModel.isFG()) {
List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
queryModel.setTableBlockInfos(tableBlockInfoList);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
index 3e6adaf..001f6c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
@@ -111,6 +111,7 @@ class SearchModeTestCase extends QueryTest with BeforeAndAfterAll {
}
test("test lucene datamap with search mode") {
+ sql("set carbon.search.enabled = true")
sql("DROP DATAMAP IF EXISTS dm ON TABLE main")
sql("CREATE DATAMAP dm ON TABLE main USING 'lucene' DMProperties('INDEX_COLUMNS'='id') ")
checkAnswer(sql("SELECT * FROM main WHERE TEXT_MATCH('id:100000')"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
----------------------------------------------------------------------
diff --git a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
index 4bfadce..0a3110e 100644
--- a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
+++ b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
@@ -207,6 +207,7 @@ public class SearchRequestHandler {
LOG.info(String.format("[SearchId:%d] pruned using FG DataMap, pruned blocks: %d", queryId,
blockToRead.size()));
queryModel.setTableBlockInfos(blockToRead);
+ queryModel.setFG(true);
return queryModel;
}