You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/02/21 01:18:20 UTC
[1/2] incubator-carbondata git commit: Fix load fail with unsafe
enabled and with bigdecimal datatypes
Repository: incubator-carbondata
Updated Branches:
refs/heads/master a6c8d2a79 -> 716517619
Fix load fail with unsafe enabled and with bigdecimal datatypes
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/aca59cea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/aca59cea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/aca59cea
Branch: refs/heads/master
Commit: aca59cea38df7a5509d1f4c96817c2ef7cd7dd9a
Parents: a6c8d2a
Author: ravipesala <ra...@gmail.com>
Authored: Fri Feb 3 19:35:13 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 21 09:06:12 2017 +0800
----------------------------------------------------------------------
.../bigdecimal/TestDimensionWithDecimalDataType.scala | 14 ++++++++++----
.../newflow/sort/unsafe/UnsafeCarbonRowPage.java | 3 +--
.../unsafe/holder/UnsafeSortTempFileChunkHolder.java | 3 +--
.../unsafe/merger/UnsafeIntermediateFileMerger.java | 5 +----
.../store/CarbonFactDataHandlerColumnar.java | 1 +
5 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
index 743b39b..e267763 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
@@ -39,10 +39,16 @@ class TestDimensionWithDecimalDataType extends QueryTest with BeforeAndAfterAll
sql(s"LOAD DATA local inpath '$resourcesPath/decimalDataWithoutHeader.csv' INTO table hiveTable")
}
-// test("test detail query on dimension column with decimal data type") {
-// checkAnswer(sql("select salary from carbonTable order by salary"),
-// sql("select salary from hiveTable order by salary"))
-// }
+ test("test unsafe with bigdecimal") {
+ sql("drop table if exists unsafecarbonTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
+ sql("CREATE TABLE IF NOT EXISTS unsafecarbonTable (ID Int, date Timestamp, country String, name String, phonetype String, serialname String, salary Decimal(19,2))STORED BY 'org.apache.carbondata.format'")
+ sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/decimalDataWithHeader.csv' into table unsafecarbonTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
+ sql("drop table if exists unsafecarbonTable")
+ }
test("test aggregate query on dimension column with decimal data type") {
checkAnswer(sql("select sum(salary) from carbonTable"),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index 561ad38..60af5d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -196,9 +196,8 @@ public class UnsafeCarbonRowPage {
size += 2;
CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- BigDecimal val = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
size += bigDecimalInBytes.length;
- rowToFill[dimensionSize + mesCount] = val;
+ rowToFill[dimensionSize + mesCount] = bigDecimalInBytes;
}
} else {
rowToFill[dimensionSize + mesCount] = null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 0c89821..60f259e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sortandgroupby.sortdata.NewRowComparator;
@@ -333,7 +332,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
short aShort = stream.readShort();
byte[] bigDecimalInBytes = new byte[aShort];
stream.readFully(bigDecimalInBytes);
- row[dimensionCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
+ row[dimensionCount + mesCount] = bigDecimalInBytes;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index c6b5e9a..ed8023b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -23,7 +23,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.AbstractQueue;
import java.util.Arrays;
@@ -34,7 +33,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.newflow.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.newflow.sort.unsafe.holder.UnsafeSortTempFileChunkHolder;
@@ -320,8 +318,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
rowData.putLong(size, val);
size += 8;
} else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
+ byte[] bigDecimalInBytes = (byte[]) value;
rowData.putShort(size, (short)bigDecimalInBytes.length);
size += 2;
for (int i = 0; i < bigDecimalInBytes.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 7c48cec..bf66700 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -1540,6 +1540,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
blockletDataHolder.put(nodeHolder, indexInNodeHolderArray);
return null;
} catch (Throwable throwable) {
+ LOGGER.error(throwable, "Error in producer");
consumerExecutorService.shutdownNow();
resetBlockletProcessingCount();
throw new CarbonDataWriterException(throwable.getMessage(), throwable);
[2/2] incubator-carbondata git commit: [CARBONDATA-690]Fix load fail
with unsafe enabled and with bigdecimal datatypes This closes #585
Posted by ja...@apache.org.
[CARBONDATA-690]Fix load fail with unsafe enabled and with bigdecimal datatypes This closes #585
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/71651761
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/71651761
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/71651761
Branch: refs/heads/master
Commit: 7165176196c52b980e2befc4a2fc4fff70f4fac8
Parents: a6c8d2a aca59ce
Author: jackylk <ja...@huawei.com>
Authored: Tue Feb 21 09:18:01 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 21 09:18:01 2017 +0800
----------------------------------------------------------------------
.../bigdecimal/TestDimensionWithDecimalDataType.scala | 14 ++++++++++----
.../newflow/sort/unsafe/UnsafeCarbonRowPage.java | 3 +--
.../unsafe/holder/UnsafeSortTempFileChunkHolder.java | 3 +--
.../unsafe/merger/UnsafeIntermediateFileMerger.java | 5 +----
.../store/CarbonFactDataHandlerColumnar.java | 1 +
5 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------