You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:04 UTC
[17/50] [abbrv] incubator-carbondata git commit: [Bug] Timestamp with
different format compare to spark date format will fail to provide filter
result. this is because while (#754)
[Bug] Timestamp with different format compare to spark date format will fail to provide filter result. this is because while (#754)
querying the engine is trying to get the direct surrogate based in user defined time format in properties file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5b6081d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5b6081d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5b6081d7
Branch: refs/heads/master
Commit: 5b6081d7b203411fcc2e4b941c6f67f62d8c2878
Parents: 73975b0
Author: sujith71955 <su...@gmail.com>
Authored: Sun Jun 26 22:09:17 2016 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Sun Jun 26 19:39:17 2016 +0530
----------------------------------------------------------------------
.../DirectDictionaryGenerator.java | 13 ++++++++
.../TimeStampDirectDictionaryGenerator.java | 32 ++++++++++++++++----
.../query/expression/ExpressionResult.java | 9 ++++--
.../visitor/CustomTypeDictionaryVisitor.java | 4 ++-
.../src/test/resources/data2_DiffTimeFormat.csv | 4 +++
.../filterexpr/FilterProcessorTestCase.scala | 25 +++++++++++++++
6 files changed, 77 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
index 476d09f..145c89a 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
@@ -40,4 +40,17 @@ public interface DirectDictionaryGenerator {
*/
Object getValueFromSurrogate(int key);
+ /**
+ * The method generate and returns the dictionary / surrogate key for direct dictionary column
+ * This Method is called while executing filter queries for getting direct surrogate members.
+ * Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format no matter
+ * in which format the data is been stored, so while retrieving the direct surrogate value for
+ * filter member first it should be converted in date form as per above format and needs to
+ * retrieve time stamp.
+ *
+ * @param member The member string value
+ * @return returns dictionary/ surrogate value
+ */
+ int generateDirectSurrogateKey(String memberStr, String format);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 8493991..97baf3d 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -40,12 +40,6 @@ import static org.carbondata.core.keygenerator.directdictionary.timestamp.TimeSt
public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator {
/**
- * Logger instance
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
-
- /**
* The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis.
*/
public static final long granularityFactor;
@@ -55,6 +49,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
* customized the start of position. for example "January 1, 2000"
*/
public static final long cutOffTimeStamp;
+ /**
+ * Logger instance
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
/**
* initialization block for granularityFactor and cutOffTimeStamp
@@ -119,6 +118,27 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
return 1;
}
+ return getDirectSurrogateForMember(memberStr, timeParser);
+ }
+
+ /**
+ * The method take member String as input and converts
+ * and returns the dictionary key
+ *
+ * @param memberStr date format string
+ * @return dictionary value
+ */
+ public int generateDirectSurrogateKey(String memberStr, String format) {
+ SimpleDateFormat timeParser = new SimpleDateFormat(format);
+ timeParser.setLenient(false);
+ if (null == memberStr || memberStr.trim().isEmpty() || memberStr
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ return 1;
+ }
+ return getDirectSurrogateForMember(memberStr, timeParser);
+ }
+
+ private int getDirectSurrogateForMember(String memberStr, SimpleDateFormat timeParser) {
Date dateToStr = null;
try {
dateToStr = timeParser.parse(memberStr);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java b/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
index 1f258e2..067f2d3 100644
--- a/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
+++ b/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
@@ -243,9 +243,12 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
try {
switch (this.getDataType()) {
case StringType:
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ // Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format
+ // no matter in which format the data is been stored, so while retrieving the direct
+ // surrogate value for filter member first it should be converted in date form as per
+ // above format and needs to retrieve time stamp.
+ SimpleDateFormat parser =
+ new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
Date dateToStr;
try {
dateToStr = parser.parse(value.toString());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 0aec1c1..81808e8 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.carbondata.query.expression.ColumnExpression;
@@ -71,7 +72,8 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
.getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
// Reading the dictionary value direct
for (String filterMember : evaluateResultListFinal) {
- surrogates.add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember));
+ surrogates.add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
}
Collections.sort(surrogates);
DimColumnFilterInfo columnFilterInfo = null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/data2_DiffTimeFormat.csv b/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
new file mode 100644
index 0000000..c5fe230
--- /dev/null
+++ b/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
@@ -0,0 +1,4 @@
+ID,date,country,name,phonetype,serialname,salary
+4,07-10-2014 00:00:00,china,aaa4,phone2435,ASD66902,4
+8,07-20-2014 00:00:00,china,aaa5,phone2441,ASD90633,10
+6,07-25-2014 00:00:00,china,aaa6,phone294,ASD59961,15005
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
index 325800b..41930df 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
@@ -40,10 +40,34 @@ class FilterProcessorTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists filtertestTables")
sql("drop table if exists filtertestTablesWithDecimal")
sql("drop table if exists filtertestTablesWithNull")
+ sql("drop table if exists filterWithTimeStamp")
sql("CREATE TABLE filtertestTables (ID int, date Timestamp, country String, " +
"name String, phonetype String, serialname String, salary int) " +
"STORED BY 'org.apache.carbondata.format'"
)
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "MM-dd-yyyy HH:mm:ss")
+
+ sql("CREATE TABLE filterWithTimeStamp (ID int, date Timestamp, country String, " +
+ "name String, phonetype String, serialname String, salary int) " +
+ "STORED BY 'org.apache.carbondata.format'"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH './src/test/resources/data2_DiffTimeFormat.csv' INTO TABLE " +
+ s"filterWithTimeStamp " +
+ s"OPTIONS('DELIMITER'= ',', " +
+ s"'FILEHEADER'= '')"
+ )
+
+ test("Time stamp filter with diff time format for load ") {
+ checkAnswer(
+ sql("select date from filterWithTimeStamp where date > '2014-07-10 00:00:00'"),
+ Seq(Row(Timestamp.valueOf("2014-07-20 00:00:00.0")),
+ Row(Timestamp.valueOf("2014-07-25 00:00:00.0"))
+ )
+ )
+ }
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
sql(
@@ -79,6 +103,7 @@ class FilterProcessorTestCase extends QueryTest with BeforeAndAfterAll {
)
}
+
test("Is not null filter") {
checkAnswer(
sql("select id from filtertestTablesWithNull " + "where id is not null"),