You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by jb...@apache.org on 2016/06/23 14:16:02 UTC
[14/56] [abbrv] incubator-carbondata git commit: [Issue - 685]
Support null or empty value for timestamp column (#686)
[Issue - 685] Support null or empty value for timestamp column (#686)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/91951acf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/91951acf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/91951acf
Branch: refs/heads/master
Commit: 91951acf2f89a0f5d0cf340a273f5ede8a08afb2
Parents: f479e7e
Author: Mohammad Shahid Khan <mo...@gmail.com>
Authored: Fri Jun 17 23:43:25 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Fri Jun 17 23:43:25 2016 +0530
----------------------------------------------------------------------
.../TimeStampDirectDictionaryGenerator.java | 21 +++--
.../DirectDictionaryDimensionAggregator.java | 21 +++--
.../resolver/ConditionalFilterResolverImpl.java | 9 +-
.../spark/src/test/resources/datasamplenull.csv | 3 +
.../TimestampDataTypeNullDataTest.scala | 92 ++++++++++++++++++++
5 files changed, 127 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index d37dcf1..060adcb 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -1,4 +1,3 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -115,22 +114,27 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
timeParser.setLenient(false);
+ if (null == memberStr || memberStr.trim().isEmpty() || memberStr
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ return 1;
+ }
Date dateToStr = null;
try {
dateToStr = timeParser.parse(memberStr);
} catch (ParseException e) {
- LOGGER.error("Cannot convert" + TIMESTAMP.toString() + " to Time/Long type value"
- + e.getMessage());
+ LOGGER.error(
+ "Cannot convert" + TIMESTAMP.toString() + " to Time/Long type value" + e.getMessage());
}
+ //adding +2 to reserve the first cuttOffDiff value for null or empty date
if (null == dateToStr) {
return -1;
} else {
if (cutOffTimeStamp >= 0) {
int keyValue = (int) ((dateToStr.getTime() - cutOffTimeStamp) / granularityFactor);
- return keyValue < 0 ? -1 : keyValue;
+ return keyValue < 0 ? -1 : keyValue + 2;
} else {
int keyValue = (int) (dateToStr.getTime() / granularityFactor);
- return keyValue < 0 ? -1 : keyValue;
+ return keyValue < 0 ? -1 : keyValue + 2;
}
}
}
@@ -142,11 +146,14 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
* @return member value/actual value Date
*/
@Override public Object getValueFromSurrogate(int key) {
+ if (key == 1) {
+ return null;
+ }
long timeStamp = 0;
if (cutOffTimeStamp >= 0) {
- timeStamp = (key * granularityFactor + cutOffTimeStamp);
+ timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
} else {
- timeStamp = key * granularityFactor;
+ timeStamp = (key - 2) * granularityFactor;
}
return timeStamp * 1000L;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java b/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
index 971e4cc..8794021 100644
--- a/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
@@ -19,6 +19,7 @@
package org.carbondata.query.carbon.aggregator.dimension.impl;
import java.nio.ByteBuffer;
+import java.sql.Timestamp;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -96,17 +97,19 @@ public class DirectDictionaryDimensionAggregator implements DimensionDataAggrega
MeasureAggregator[] aggeragtor) {
byte[] dimensionData = scannedResult.getDimensionKey(blockIndex);
int surrogateKey = CarbonUtil.getSurrogateKey(dimensionData, buffer);
- Object dataBasedOnDataType =
- (long) directDictionaryGenerator.getValueFromSurrogate(surrogateKey) / 1000;
+ Object valueFromSurrogate = directDictionaryGenerator.getValueFromSurrogate(surrogateKey);
+ if (null != valueFromSurrogate) {
+ Timestamp dataBasedOnDataType = new Timestamp((long) valueFromSurrogate / 1000);
- if (actualTypeAggregatorIndex.length > 0) {
- for (int j = 0; j < actualTypeAggregatorIndex.length; j++) {
- aggeragtor[aggregatorStartIndex + actualTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+ if (actualTypeAggregatorIndex.length > 0) {
+ for (int j = 0; j < actualTypeAggregatorIndex.length; j++) {
+ aggeragtor[aggregatorStartIndex + actualTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+ }
}
- }
- if (numberTypeAggregatorIndex.length > 0) {
- for (int j = 0; j < numberTypeAggregatorIndex.length; j++) {
- aggeragtor[aggregatorStartIndex + numberTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+ if (numberTypeAggregatorIndex.length > 0) {
+ for (int j = 0; j < numberTypeAggregatorIndex.length; j++) {
+ aggeragtor[aggregatorStartIndex + numberTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
index 4897736..59ef1be 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
@@ -24,6 +24,7 @@ import java.util.SortedMap;
import org.carbondata.core.carbon.AbsoluteTableIdentifier;
import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
import org.carbondata.query.carbonfilterinterface.FilterExecuterType;
import org.carbondata.query.expression.ColumnExpression;
@@ -80,9 +81,11 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
// column expression.
// we need to check if the other expression contains column
// expression or not in depth.
- if (FilterUtil.checkIfExpressionContainsColumn(rightExp)||
- FilterUtil.isExpressionNeedsToResolved(rightExp,isIncludeFilter) &&
- columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)){
+ CarbonDimension dimension = columnExpression.getDimension();
+ if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
+ || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) &&
+ dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
+ .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
isExpressionResolve = true;
} else {
//Visitor pattern is been used in this scenario inorder to populate the
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/integration/spark/src/test/resources/datasamplenull.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datasamplenull.csv b/integration/spark/src/test/resources/datasamplenull.csv
new file mode 100644
index 0000000..475bbe1
--- /dev/null
+++ b/integration/spark/src/test/resources/datasamplenull.csv
@@ -0,0 +1,3 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000
+2,,china,aaa2,phone756,ASD42892,15001
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
new file mode 100644
index 0000000..c8c1f81
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.spark.testsuite.directdictionary
+
+import java.io.File
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.hive.HiveContext
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants
+import org.carbondata.core.util.CarbonProperties
+import org.scalatest.BeforeAndAfterAll
+
+
+/**
+ * Test Class for detailed query on timestamp datatypes
+ *
+ *
+ */
+class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
+ var oc: HiveContext = _
+
+ override def beforeAll {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00.0")
+ CarbonProperties.getInstance()
+ .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY,
+ TimeStampGranularityConstants.TIME_GRAN_SEC.toString
+ )
+ sql(
+ """CREATE TABLE IF NOT EXISTS timestampTyeNullData
+ (ID Int, date Timestamp, country String,
+ name String, phonetype String, serialname String, salary Int)
+ STORED BY 'org.apache.carbondata.format'"""
+ )
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ var csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv"
+ sql("LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE timestampTyeNullData").collect();
+
+ } catch {
+ case x: Throwable => CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+ }
+
+ test("SELECT max(date) FROM timestampTyeNullData where date is not null") {
+ checkAnswer(
+ sql("SELECT max(date) FROM timestampTyeNullData where date is not null"),
+ Seq(Row(Timestamp.valueOf("2015-01-23 00:07:00.0"))
+ )
+ )
+ }
+ test("SELECT * FROM timestampTyeNullData where date is null") {
+ checkAnswer(
+ sql("SELECT date FROM timestampTyeNullData where date is null"),
+ Seq(Row(null)
+ ))
+ }
+
+ override def afterAll {
+ sql("drop table timestampTyeNullData")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
+ }
+
+}
\ No newline at end of file