You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/05/15 05:06:22 UTC
[09/19] carbondata git commit: fix sort_columns issue
fix sort_columns issue
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f5c6f288
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f5c6f288
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f5c6f288
Branch: refs/heads/master
Commit: f5c6f2880d4db863494ad9491c24438010c5340d
Parents: 81149f6
Author: QiangCai <qi...@qq.com>
Authored: Sat Apr 8 22:10:17 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:12 2017 +0800
----------------------------------------------------------------------
.../executer/RowLevelFilterExecuterImpl.java | 3 ++-
.../RowLevelRangeFilterResolverImpl.java | 5 +++-
.../apache/carbondata/core/util/ByteUtil.java | 28 +++++++++++---------
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++--
.../CarbonRowDataWriterProcessorStepImpl.java | 4 +--
6 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8e7c95f..38204e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -254,7 +254,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
memberBytes = null;
}
record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension());
+ .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
} else {
continue;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2a1af65..6d4e528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
@@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
continue;
}
- filterValuesList.add(result.getString().getBytes());
+ filterValuesList.add(DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(),
+ result.getDataType()));
} catch (FilterIllegalMemberException e) {
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f3f0fcd..70ddec9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -404,6 +404,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(short val) {
+ val = (short)(val ^ Short.MIN_VALUE);
byte[] b = new byte[SIZEOF_SHORT];
b[1] = (byte) val;
val >>= 8;
@@ -423,20 +424,21 @@ public final class ByteUtil {
if (length != SIZEOF_SHORT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
}
+ short n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Short.reverseBytes(
+ n = Short.reverseBytes(
CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- short n = 0;
+
n ^= bytes[offset] & 0xFF;
n <<= 8;
n ^= bytes[offset + 1] & 0xFF;
- return n;
}
+ return (short)(n ^ Short.MIN_VALUE);
}
/**
@@ -446,6 +448,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(int val) {
+ val = val ^ Integer.MIN_VALUE;
byte[] b = new byte[4];
for (int i = 3; i > 0; i--) {
b[i] = (byte) val;
@@ -467,21 +470,21 @@ public final class ByteUtil {
if (length != SIZEOF_INT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
}
+ int n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Integer.reverseBytes(
+ n = Integer.reverseBytes(
CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- int n = 0;
for (int i = offset; i < (offset + length); i++) {
n <<= 8;
n ^= bytes[i] & 0xFF;
}
- return n;
}
+ return n ^ Integer.MIN_VALUE;
}
/**
@@ -513,6 +516,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(long val) {
+ val = val ^ Long.MIN_VALUE;
byte[] b = new byte[8];
for (int i = 7; i > 0; i--) {
b[i] = (byte) val;
@@ -529,21 +533,21 @@ public final class ByteUtil {
if (length != SIZEOF_LONG || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
}
+ long l = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Long.reverseBytes(
+ l = Long.reverseBytes(
CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- long l = 0;
for (int i = offset; i < offset + length; i++) {
l <<= 8;
l ^= bytes[i] & 0xFF;
}
- return l;
}
+ return l ^ Long.MIN_VALUE;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 2e16b0f..80c9e72 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
}
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
- DataType actualDataType) throws Throwable {
+ DataType actualDataType) {
switch (actualDataType) {
case STRING:
return ByteUtil.toBytes(dimensionValue);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 88b0421..afdca21 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order by empno"), sql("select * from origintable1 where empno > 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15 order by empno"), sql("select * from origintable1 where empno >= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <> 15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16, 17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"), sql("select * from origintable1 where empno is null order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"), sql("select * from origintable1 where empno is not null order by empno"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
} finally {
defaultLoadingProperties
}
}
-
-
+
override def afterAll = {
dropTable
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f5c6f288/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 929a09e..c50f335 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
- model.getCarbonDataFileAttributes()
- .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
CarbonFactHandler dataHandler = null;
boolean rowsNotExist = true;
while (iterator.hasNext()) {