You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2019/05/16 19:05:44 UTC
[carbondata] 11/22: [CARBONDATA-3359]Fix data mismatch issue for
decimal column after delete operation
This is an automated email from the ASF dual-hosted git repository.
ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git
commit f7cdb47e7535c7543147bf96f42cfbc14b36b082
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Thu Apr 25 15:16:35 2019 +0530
[CARBONDATA-3359]Fix data mismatch issue for decimal column after delete operation
Problem:
after delete operation is performed, the decimal column data is wrong. This is because, during filling vector for decimal column,
we were not considering the deleted rows if present any, we were filling all the row data for decimal.
Solution
in case of decimal, get the vector from ColumnarVectorWrapperDirectFactory and then put data, which will take care of the deleted rows
This closes #3189
---
.../metadata/datatype/DecimalConverterFactory.java | 55 +++++++++++++---------
.../src/test/resources/decimalData.csv | 4 ++
.../testsuite/iud/DeleteCarbonTableTestCase.scala | 17 +++++++
3 files changed, 54 insertions(+), 22 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 9793c38..2e155f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -102,13 +103,13 @@ public final class DecimalConverterFactory {
return BigDecimal.valueOf((Long) valueToBeConverted, scale);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
// TODO we need to find way to directly set to vector with out conversion. This way is very
// inefficient.
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (!(valuesToBeConverted instanceof byte[])) {
throw new UnsupportedOperationException("This object type " + valuesToBeConverted.getClass()
+ " is not supported in this method");
@@ -116,7 +117,7 @@ public final class DecimalConverterFactory {
byte[] data = (byte[]) valuesToBeConverted;
if (pageType == DataTypes.BYTE) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal.valueOf(data[i], scale);
@@ -128,7 +129,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.SHORT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -142,7 +143,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.SHORT_INT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -156,7 +157,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.INT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -170,7 +171,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.LONG) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -261,18 +262,18 @@ public final class DecimalConverterFactory {
return new BigDecimal(bigInteger, scale);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (scale < newMeasureScale) {
scale = newMeasureScale;
}
if (valuesToBeConverted instanceof byte[][]) {
byte[][] data = (byte[][]) valuesToBeConverted;
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigInteger bigInteger = new BigInteger(data[i]);
@@ -307,15 +308,15 @@ public final class DecimalConverterFactory {
return DataTypeUtil.byteToBigDecimal((byte[]) valueToBeConverted);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (valuesToBeConverted instanceof byte[][]) {
byte[][] data = (byte[][]) valuesToBeConverted;
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = DataTypeUtil.byteToBigDecimal(data[i]);
@@ -337,6 +338,16 @@ public final class DecimalConverterFactory {
}
}
+ private static CarbonColumnVector getCarbonColumnVector(ColumnVectorInfo vectorInfo,
+ BitSet nullBitSet) {
+ CarbonColumnVector vector = vectorInfo.vector;
+ BitSet deletedRows = vectorInfo.deletedRows;
+ vector = ColumnarVectorWrapperDirectFactory
+ .getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBitSet, deletedRows,
+ true, false);
+ return vector;
+ }
+
public DecimalConverter getDecimalConverter(int precision, int scale) {
if (precision < 0) {
return new LVBytesDecimalConverter();
diff --git a/integration/spark-common-test/src/test/resources/decimalData.csv b/integration/spark-common-test/src/test/resources/decimalData.csv
new file mode 100644
index 0000000..8cbbd21
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/decimalData.csv
@@ -0,0 +1,4 @@
+smallIntField,intField,bigIntField,floatField,doubleField,decimalField,timestampField,dateField,stringField,varcharField,charField,arrayField,structField
+-1,-1,-1,-1.1,-1.1,-1.1234,2017-06-11 00:00:01,2017-06-11,abc1,abcd1,abcde1,a$b$c$1,a$b$1
+2,2,2,2.1,2.1,2.1234,2017-06-12 23:59:02,2017-06-12,abc2,abcd2,abcde2,a$b$c$2,a$b$2
+3,3,3,3.1,3.1,3.1234,2017-06-13 23:59:03,2017-06-13,abc3,abcd3,abcde3,a$b$c$3,a$b$3
\ No newline at end of file
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 2f95133..f26283b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -344,6 +344,23 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists test_dm_index")
}
+ test("test delete on table with decimal column") {
+ sql("drop table if exists decimal_table")
+ sql(
+ s"""create table decimal_table(smallIntField smallInt,intField int,bigIntField bigint,floatField float,
+ doubleField double,decimalField decimal(25, 4),timestampField timestamp,dateField date,stringField string,
+ varcharField varchar(10),charField char(10))stored as carbondata
+ """.stripMargin)
+ sql(s"load data local inpath '$resourcesPath/decimalData.csv' into table decimal_table")
+ val frame = sql("select decimalfield from decimal_table where smallIntField = -1 or smallIntField = 3")
+ sql(s"delete from decimal_table where smallIntField = 2")
+ checkAnswer(frame, Seq(
+ Row(-1.1234),
+ Row(3.1234)
+ ))
+ sql("drop table if exists decimal_table")
+ }
+
override def afterAll {
sql("use default")
sql("drop database if exists iud_db cascade")