You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/05/17 14:30:35 UTC
[22/50] [abbrv] carbondata git commit: [CARBONDATA-2455]Fix _System
Folder creation and lucene AND, OR, NOT Filter fix
[CARBONDATA-2455]Fix _System Folder creation and lucene AND,OR,NOT Filter fix
This closes #2281
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61afa42d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61afa42d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61afa42d
Branch: refs/heads/spark-2.3
Commit: 61afa42daafa03a7e8e0eb980cb6d667d6304785
Parents: 6b70b7e
Author: Indhumathi27 <in...@gmail.com>
Authored: Tue May 8 15:36:47 2018 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Fri May 11 09:29:19 2018 +0800
----------------------------------------------------------------------
.../schema/table/DiskBasedDMSchemaStorageProvider.java | 2 +-
.../carbondata/datamap/lucene/LuceneDataMapWriter.java | 1 -
.../datamap/lucene/LuceneFineGrainDataMap.java | 3 +--
.../cluster/sdv/generated/LuceneTestCase.scala | 7 +++----
.../datamap/lucene/LuceneFineGrainDataMapSuite.scala | 11 +++++------
5 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
index e1a929c..4ebbcd0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
@@ -55,7 +55,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro
private Set<DataMapSchema> dataMapSchemas = new HashSet<>();
public DiskBasedDMSchemaStorageProvider(String storePath) {
- this.storePath = storePath;
+ this.storePath = CarbonUtil.checkAndAppendHDFSUrl(storePath);
this.mdtFilePath = storePath + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
index 12fa1ef..3615936 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
@@ -274,7 +274,6 @@ public class LuceneDataMapWriter extends DataMapWriter {
}
} else if (type == DataTypes.STRING) {
byte[] value = (byte[]) data;
- // TODO: how to get string value
String strValue = null;
try {
strValue = new String(value, 2, value.length - 2, "UTF-8");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
index 8e457b7..f8d1b12 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
@@ -205,8 +205,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap {
queryParser.setAllowLeadingWildcard(true);
Query query;
try {
- // always send lowercase string to lucene as it is case sensitive
- query = queryParser.parse(strQuery.toLowerCase());
+ query = queryParser.parse(strQuery);
} catch (ParseException e) {
String errorMessage = String.format(
"failed to filter block with query %s, detail is %s", strQuery, e.getMessage());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
index d111594..54d0d11 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
@@ -110,14 +110,13 @@ class LuceneTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*')"),
sql("select * from datamap_main where country like 'ch%'"))
checkAnswer(sql(
- "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*') AND TEXT_MATCH('name:aa*')"),
+ "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch* AND name:aa*')"),
sql("select * from datamap_main where country like 'ch%' and name like 'aa%'"))
checkAnswer(sql(
- "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* or name:aa*')"),
+ "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR name:aa*')"),
sql("select * from datamap_main where country like 'u%'or name like 'aa%'"))
checkAnswer(sql(
- "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u*') OR TEXT_MATCH('name:aaa1*') AND " +
- "TEXT_MATCH('name:aaa2*')"),
+ "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR (name:aaa1* AND name:aaa2*)')"),
sql(
"select * from datamap_main where country like 'u%' OR name like 'aaa1%' AND name like " +
"'aaa2%'"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 9981ff5..6d2eb3f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -290,7 +290,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
checkAnswer(sql(
- "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0*') AND TEXT_MATCH(' city:c0*')"),
+ "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0* AND city:c0*')"),
sql("select * from datamap_test_table where name like 'n0%' and city like 'c0%'"))
sql("drop datamap if exists dm on table datamap_test_table")
}
@@ -311,7 +311,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
checkAnswer(sql(
- "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') or TEXT_MATCH('city:c01*')"),
+ "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR city:c01*')"),
sql("select * from datamap_test_table where name like 'n1%' or city like 'c01%'"))
sql("drop datamap if exists dm on table datamap_test_table")
}
@@ -332,8 +332,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
checkAnswer(sql(
- "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') OR TEXT_MATCH ('city:c01*') " +
- "AND TEXT_MATCH('city:C02*')"),
+ "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR (city:c01* AND city:c02*)')"),
sql(
"select * from datamap_test_table where name like 'n1%' OR city like 'c01%' and city like" +
" 'c02%'"))
@@ -488,7 +487,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql("select *from datamap_test_table where name like'n1%' AND not name like 'n2%'"))
//check NOT filter with TEXTMATCH wildcard-search using AND on different columns
checkAnswer(sql(
- "select *from datamap_test_table where TEXT_MATCH('name:n1*')AND TEXT_MATCH('city:c01* NOT " +
+ "select *from datamap_test_table where TEXT_MATCH('name:n1* AND city:c01* NOT " +
"c02*')"),
sql("select *from datamap_test_table where name like'n1%' AND not city='c02%'"))
sql("drop datamap if exists dm on table datamap_test_table")
@@ -716,7 +715,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql(s"SELECT * FROM datamap_test5 WHERE city='c020'"))
sql("DROP TABLE IF EXISTS datamap_test5")
}
-
+
test("test text_match on normal table") {
sql("DROP TABLE IF EXISTS table1")
sql(