You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/05/02 08:28:32 UTC
[carbondata] branch master updated: [CARBONDATA-3359]Fix data
mismatch issue for decimal column after delete operation
This is an automated email from the ASF dual-hosted git repository.
kumarvishal09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new eb7a833 [CARBONDATA-3359]Fix data mismatch issue for decimal column after delete operation
eb7a833 is described below
commit eb7a8335013957c9a615a48c7304b7968a2f7e24
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Thu Apr 25 15:16:35 2019 +0530
[CARBONDATA-3359]Fix data mismatch issue for decimal column after delete operation
Problem:
after delete operation is performed, the decimal column data is wrong. This is because, during filling vector for decimal column,
we were not considering the deleted rows if present any, we were filling all the row data for decimal.
Solution
in case of decimal, get the vector from ColumnarVectorWrapperDirectFactory and then put data, which will take care of the deleted rows
This closes #3189
---
.../metadata/datatype/DecimalConverterFactory.java | 55 +++++++++++++---------
.../src/test/resources/decimalData.csv | 4 ++
.../testsuite/iud/DeleteCarbonTableTestCase.scala | 17 +++++++
3 files changed, 54 insertions(+), 22 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 9793c38..2e155f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.result.vector.impl.directread.ColumnarVectorWrapperDirectFactory;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -102,13 +103,13 @@ public final class DecimalConverterFactory {
return BigDecimal.valueOf((Long) valueToBeConverted, scale);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
// TODO we need to find way to directly set to vector with out conversion. This way is very
// inefficient.
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (!(valuesToBeConverted instanceof byte[])) {
throw new UnsupportedOperationException("This object type " + valuesToBeConverted.getClass()
+ " is not supported in this method");
@@ -116,7 +117,7 @@ public final class DecimalConverterFactory {
byte[] data = (byte[]) valuesToBeConverted;
if (pageType == DataTypes.BYTE) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal.valueOf(data[i], scale);
@@ -128,7 +129,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.SHORT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -142,7 +143,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.SHORT_INT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -156,7 +157,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.INT) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -170,7 +171,7 @@ public final class DecimalConverterFactory {
}
} else if (pageType == DataTypes.LONG) {
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = BigDecimal
@@ -261,18 +262,18 @@ public final class DecimalConverterFactory {
return new BigDecimal(bigInteger, scale);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (scale < newMeasureScale) {
scale = newMeasureScale;
}
if (valuesToBeConverted instanceof byte[][]) {
byte[][] data = (byte[][]) valuesToBeConverted;
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigInteger bigInteger = new BigInteger(data[i]);
@@ -307,15 +308,15 @@ public final class DecimalConverterFactory {
return DataTypeUtil.byteToBigDecimal((byte[]) valueToBeConverted);
}
- @Override public void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info,
- BitSet nullBitset, DataType pageType) {
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- int newMeasureScale = info.measure.getMeasure().getScale();
+ @Override public void fillVector(Object valuesToBeConverted, int size,
+ ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
+ CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
+ int precision = vectorInfo.measure.getMeasure().getPrecision();
+ int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
if (valuesToBeConverted instanceof byte[][]) {
byte[][] data = (byte[][]) valuesToBeConverted;
for (int i = 0; i < size; i++) {
- if (nullBitset.get(i)) {
+ if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
BigDecimal value = DataTypeUtil.byteToBigDecimal(data[i]);
@@ -337,6 +338,16 @@ public final class DecimalConverterFactory {
}
}
+ private static CarbonColumnVector getCarbonColumnVector(ColumnVectorInfo vectorInfo,
+ BitSet nullBitSet) {
+ CarbonColumnVector vector = vectorInfo.vector;
+ BitSet deletedRows = vectorInfo.deletedRows;
+ vector = ColumnarVectorWrapperDirectFactory
+ .getDirectVectorWrapperFactory(vector, vectorInfo.invertedIndex, nullBitSet, deletedRows,
+ true, false);
+ return vector;
+ }
+
public DecimalConverter getDecimalConverter(int precision, int scale) {
if (precision < 0) {
return new LVBytesDecimalConverter();
diff --git a/integration/spark-common-test/src/test/resources/decimalData.csv b/integration/spark-common-test/src/test/resources/decimalData.csv
new file mode 100644
index 0000000..8cbbd21
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/decimalData.csv
@@ -0,0 +1,4 @@
+smallIntField,intField,bigIntField,floatField,doubleField,decimalField,timestampField,dateField,stringField,varcharField,charField,arrayField,structField
+-1,-1,-1,-1.1,-1.1,-1.1234,2017-06-11 00:00:01,2017-06-11,abc1,abcd1,abcde1,a$b$c$1,a$b$1
+2,2,2,2.1,2.1,2.1234,2017-06-12 23:59:02,2017-06-12,abc2,abcd2,abcde2,a$b$c$2,a$b$2
+3,3,3,3.1,3.1,3.1234,2017-06-13 23:59:03,2017-06-13,abc3,abcd3,abcde3,a$b$c$3,a$b$3
\ No newline at end of file
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 2f95133..f26283b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -344,6 +344,23 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists test_dm_index")
}
+ test("test delete on table with decimal column") {
+ sql("drop table if exists decimal_table")
+ sql(
+ s"""create table decimal_table(smallIntField smallInt,intField int,bigIntField bigint,floatField float,
+ doubleField double,decimalField decimal(25, 4),timestampField timestamp,dateField date,stringField string,
+ varcharField varchar(10),charField char(10))stored as carbondata
+ """.stripMargin)
+ sql(s"load data local inpath '$resourcesPath/decimalData.csv' into table decimal_table")
+ val frame = sql("select decimalfield from decimal_table where smallIntField = -1 or smallIntField = 3")
+ sql(s"delete from decimal_table where smallIntField = 2")
+ checkAnswer(frame, Seq(
+ Row(-1.1234),
+ Row(3.1234)
+ ))
+ sql("drop table if exists decimal_table")
+ }
+
override def afterAll {
sql("use default")
sql("drop database if exists iud_db cascade")