You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/21 17:59:51 UTC
[24/50] [abbrv] carbondata git commit: [CARBONDATA-3081] Fixed NPE
for boolean type column with null value
[CARBONDATA-3081] Fixed NPE for boolean type column with null value
Problem: NPE thrown when boolean type column has null values.
Solution: check for null values before converting byte to boolean.
This closes #2901
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b69f0fc8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b69f0fc8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b69f0fc8
Branch: refs/heads/branch-1.5
Commit: b69f0fc87da1cd50191d055658d47c3f1ea3559b
Parents: 413cd80
Author: kunal642 <ku...@gmail.com>
Authored: Mon Nov 5 18:46:44 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Nov 21 22:43:15 2018 +0530
----------------------------------------------------------------------
.../core/metadata/datatype/DecimalType.java | 2 +-
.../util/CarbonVectorizedRecordReader.java | 19 +++++---
.../carbondata/sdk/file/CarbonReaderTest.java | 49 ++++++++++++++++++++
3 files changed, 63 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
index b4bc20c..a7f7a4e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
@@ -23,7 +23,7 @@ public class DecimalType extends DataType {
private int scale;
// create a decimal type object with specified precision and scale
- public DecimalType(int precision, int scale) {
+ DecimalType(int precision, int scale) {
super(DataTypes.DECIMAL_TYPE_ID, 8, "DECIMAL", -1);
this.precision = precision;
this.scale = scale;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
index 9d3d7d6..7720434 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonVectorizedRecordReader.java
@@ -29,7 +29,6 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.datatype.StructField;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
@@ -149,7 +148,8 @@ public class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
} else if (DataTypes.isDecimal(dataType)) {
fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
- new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()));
+ DataTypes.createDecimalType(msr.getMeasure().getPrecision(),
+ msr.getMeasure().getScale()));
} else {
fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
}
@@ -171,13 +171,20 @@ public class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
rowCount += 1;
Object[] row = new Object[carbonColumnarBatch.columnVectors.length];
for (int i = 0; i < carbonColumnarBatch.columnVectors.length; i ++) {
+ Object data = carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1);
if (carbonColumnarBatch.columnVectors[i].getType() == DataTypes.STRING
|| carbonColumnarBatch.columnVectors[i].getType() == DataTypes.VARCHAR) {
- byte[] data = (byte[]) carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1);
- row[i] = ByteUtil.toString(data, 0, data.length);
+ if (data == null) {
+ row[i] = null;
+ } else {
+ row[i] = ByteUtil.toString((byte[]) data, 0, (((byte[]) data).length));
+ }
} else if (carbonColumnarBatch.columnVectors[i].getType() == DataTypes.BOOLEAN) {
- byte data = (byte) carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1);
- row[i] = ByteUtil.toBoolean(data);
+ if (data == null) {
+ row[i] = null;
+ } else {
+ row[i] = ByteUtil.toBoolean((byte) data);
+ }
} else {
row[i] = carbonColumnarBatch.columnVectors[i].getData(batchIdx - 1);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b69f0fc8/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 0767264..9add0b9 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -2023,4 +2023,53 @@ public class CarbonReaderTest extends TestCase {
}
}
+ @Test
+ public void testReadingNullValues() {
+ String path = "./testWriteFiles";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("booleanField", DataTypes.BOOLEAN);
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 2; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ "",
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ reader.readNextRow();
+ i++;
+ }
+ assert (i == 2);
+ reader.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ } finally {
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+ }
+
}