You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2018/06/26 08:52:52 UTC
carbondata git commit: [CARBONDATA-2515][CARBONDATA-2516] fixed
Timestamp greaterthan expression and OR filter Expression issue
Repository: carbondata
Updated Branches:
refs/heads/master 4d3ecfb22 -> 53a9fa7f8
[CARBONDATA-2515][CARBONDATA-2516] fixed Timestamp greaterthan expression and OR filter Expression issue
fix OR expression and Timestamp Greater than expression issues
This closes #2334
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/53a9fa7f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/53a9fa7f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/53a9fa7f
Branch: refs/heads/master
Commit: 53a9fa7f85f0ca302c77e2e27d1791fdf616f55a
Parents: 4d3ecfb
Author: sv71294 <sv...@gmail.com>
Authored: Tue May 22 17:05:18 2018 +0530
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue Jun 26 16:52:56 2018 +0800
----------------------------------------------------------------------
.../carbondata/presto/PrestoFilterUtil.java | 109 ++++++-------------
.../integrationtest/PrestoAllDataTypeTest.scala | 53 +++++++++
2 files changed, 88 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/53a9fa7f/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
old mode 100644
new mode 100755
index 65f5eb1..adefa8b
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -60,7 +60,6 @@ import com.facebook.presto.spi.type.SmallintType;
import com.facebook.presto.spi.type.TimestampType;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.spi.type.VarcharType;
-import com.google.common.collect.ImmutableList;
import io.airlift.slice.Slice;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
@@ -171,47 +170,46 @@ public class PrestoFilterUtil {
* @return
*/
static Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint) {
- ImmutableList.Builder<Expression> filters = ImmutableList.builder();
Domain domain;
+ // final expression for the table, returned by the method after combining all the column filters (colValueExpression).
+ Expression finalFilters = null;
+
for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
// Build ColumnExpression for Expression(Carbondata)
CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
Type type = cdch.getColumnType();
-
-
DataType coltype = Spi2CarbondataTypeMapper(cdch);
Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype);
domain = originalConstraint.getDomains().get().get(c);
checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
-
List<Object> singleValues = new ArrayList<>();
- Map<Object, List<Expression>> valueExpressionMap = new HashMap<>();
+
+ // combination of multiple rangeExpression for a single column in case of multiple range Filter
+ // on single column else this is equal to rangeExpression, combined to create finalFilters
+ Expression colValueExpression = null;
+
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
if (range.isSingleValue()) {
Object value = ConvertDataByType(range.getLow().getValue(), type);
singleValues.add(value);
} else {
+ // generated for each range of column i.e. lessThan, greaterThan,
+ // there can be multiple ranges for a single column. combined to create colValueExpression
+ Expression rangeExpression = null;
if (!range.getLow().isLowerUnbounded()) {
Object value = ConvertDataByType(range.getLow().getValue(), type);
switch (range.getLow().getBound()) {
case ABOVE:
- if (type == TimestampType.TIMESTAMP) {
- //todo not now
- } else {
- GreaterThanExpression greater = new GreaterThanExpression(colExpression,
- new LiteralExpression(value, coltype));
- valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
- }
+ rangeExpression =
+ new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
break;
case EXACTLY:
- GreaterThanEqualToExpression greater =
- new GreaterThanEqualToExpression(colExpression,
- new LiteralExpression(value, coltype));
- valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
+ rangeExpression = new GreaterThanEqualToExpression(colExpression,
+ new LiteralExpression(value, coltype));
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
@@ -219,86 +217,49 @@ public class PrestoFilterUtil {
throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
}
}
+
if (!range.getHigh().isUpperUnbounded()) {
+ Expression lessThanExpression;
Object value = ConvertDataByType(range.getHigh().getValue(), type);
switch (range.getHigh().getBound()) {
case ABOVE:
throw new IllegalArgumentException("High marker should never use ABOVE bound");
case EXACTLY:
- LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression,
+ lessThanExpression = new LessThanEqualToExpression(colExpression,
new LiteralExpression(value, coltype));
- valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less);
break;
case BELOW:
- LessThanExpression less2 =
+ lessThanExpression =
new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
- valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less2);
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
}
+ rangeExpression = (rangeExpression == null ?
+ lessThanExpression :
+ new AndExpression(rangeExpression, lessThanExpression));
}
+ colValueExpression = (colValueExpression == null ?
+ rangeExpression :
+ new OrExpression(colValueExpression, rangeExpression));
}
}
+
if (singleValues.size() == 1) {
- Expression ex;
- if (coltype.equals(DataTypes.STRING)) {
- ex = new EqualToExpression(colExpression,
- new LiteralExpression(singleValues.get(0), coltype));
- } else if (coltype.equals(DataTypes.TIMESTAMP) || coltype.equals(DataTypes.DATE)) {
- Long value = (Long) singleValues.get(0);
- ex = new EqualToExpression(colExpression, new LiteralExpression(value, coltype));
- } else ex = new EqualToExpression(colExpression,
+ colValueExpression = new EqualToExpression(colExpression,
new LiteralExpression(singleValues.get(0), coltype));
- filters.add(ex);
} else if (singleValues.size() > 1) {
- ListExpression candidates = null;
- List<Expression> exs = singleValues.stream()
- .map((a) -> new LiteralExpression(a, coltype))
- .collect(toList());
- candidates = new ListExpression(exs);
- filters.add(new InExpression(colExpression, candidates));
- } else if (valueExpressionMap.size() > 0) {
- List<Expression> valuefilters = new ArrayList<>();
- Expression finalFilters = null;
- List<Expression> expressions;
- for (Map.Entry<Object, List<Expression>> entry : valueExpressionMap.entrySet()) {
- expressions = valueExpressionMap.get(entry.getKey());
- if (expressions.size() == 1) {
- finalFilters = expressions.get(0);
- } else if (expressions.size() >= 2) {
- finalFilters = new OrExpression(expressions.get(0), expressions.get(1));
- for (int i = 2; i < expressions.size(); i++) {
- finalFilters = new OrExpression(finalFilters, expressions.get(i));
- }
- }
- valuefilters.add(finalFilters);
- }
-
- if (valuefilters.size() == 1) {
- finalFilters = valuefilters.get(0);
- } else if (valuefilters.size() >= 2) {
- finalFilters = new AndExpression(valuefilters.get(0), valuefilters.get(1));
- for (int i = 2; i < valuefilters.size(); i++) {
- finalFilters = new AndExpression(finalFilters, valuefilters.get(i));
- }
- }
-
- filters.add(finalFilters);
+ List<Expression> exs =
+ singleValues.stream().map((a) -> new LiteralExpression(a, coltype)).collect(toList());
+ colValueExpression = new InExpression(colExpression, new ListExpression(exs));
}
- }
- Expression finalFilters;
- List<Expression> tmp = filters.build();
- if (tmp.size() > 1) {
- finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
- if (tmp.size() > 2) {
- for (int i = 2; i < tmp.size(); i++) {
- finalFilters = new AndExpression(finalFilters, tmp.get(i));
- }
+ if (colValueExpression != null) {
+ finalFilters = (finalFilters == null ?
+ colValueExpression :
+ new AndExpression(finalFilters, colValueExpression));
}
- } else if (tmp.size() == 1) finalFilters = tmp.get(0);
- else return null;
+ }
return finalFilters;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/53a9fa7f/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
index 2e7402a..8793765 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala
@@ -18,6 +18,7 @@
package org.apache.carbondata.presto.integrationtest
import java.io.File
+import java.sql.Timestamp
import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
import org.scalatest.{BeforeAndAfterAll, FunSuiteLike}
@@ -521,4 +522,56 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll {
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
FileFactory.createNewFile(s"$storePath/testdb/.DS_STORE",FileType.LOCAL)
}
+
+ test("test the OR operator on same column"){
+ val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT BONUS FROM TESTDB.TESTTABLE WHERE" +
+ " BONUS < 600 OR BONUS > 5000 ORDER BY BONUS")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("BONUS" -> java.math.BigDecimal.valueOf(500.4140).setScale(4)),
+ Map("BONUS" -> java.math.BigDecimal.valueOf(500.5900).setScale(4)),
+ Map("BONUS" -> java.math.BigDecimal.valueOf(500.8800).setScale(4)),
+ Map("BONUS" -> java.math.BigDecimal.valueOf(500.9900).setScale(4)),
+ Map("BONUS" -> java.math.BigDecimal.valueOf(5000.9990).setScale(4)),
+ Map("BONUS" -> java.math.BigDecimal.valueOf(9999.9990).setScale(4)))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test the AND, OR operator on same column"){
+ val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ " SHORTFIELD > 4 AND (SHORTFIELD < 10 or SHORTFIELD > 15) ORDER BY SHORTFIELD")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("SHORTFIELD" -> 8),
+ Map("SHORTFIELD" -> 18))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test the OR operator with multiple AND on same column"){
+ val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ " (SHORTFIELD > 1 AND SHORTFIELD < 5) OR (SHORTFIELD > 10 AND SHORTFIELD < 15) ORDER BY SHORTFIELD")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("SHORTFIELD" -> 4),
+ Map("SHORTFIELD" -> 11),
+ Map("SHORTFIELD" -> 12))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test the OR, AND operator with on Different column"){
+ val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT SHORTFIELD FROM TESTDB.TESTTABLE WHERE" +
+ " ID < 7 AND (SHORTFIELD < 5 OR SHORTFIELD > 15) ORDER BY SHORTFIELD")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("SHORTFIELD" -> 4),
+ Map("SHORTFIELD" -> 18))
+ assert(actualResult.equals(expectedResult))
+ }
+
+ test("test the Timestamp greaterthan expression"){
+ val actualResult: List[Map[String, Any]] = PrestoServer.executeQuery("SELECT DOB FROM TESTDB.TESTTABLE" +
+ " WHERE DOB > timestamp '2016-01-01 00:00:00.0' order by DOB")
+ val expectedResult: List[Map[String, Any]] = List(
+ Map("DOB" -> new Timestamp(new java.util.Date(2016-1900,1-1,14,15,7,9).getTime)),
+ Map("DOB" -> new Timestamp(new java.util.Date(2016-1900,4-1,14,15,0,9).getTime)))
+ assert(actualResult.equals(expectedResult))
+ }
+
+
}
\ No newline at end of file