You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/04/30 12:40:21 UTC
carbondata git commit: [CARBONDATA-2411] infinite loop when sdk
writer throws Exception
Repository: carbondata
Updated Branches:
refs/heads/master 5d55b83c1 -> 3ea2a1dd5
[CARBONDATA-2411] infinite loop when sdk writer throws Exception
Exception is not handled when Object is cast to String[] in writer method of
CarbonWriter so no Exception thrown to caller and data-loading threads is not interrupted which is leading complete sdk
application to infinite loop ( Or data loading max timeout 2 Day).
Throw Exception to caller and one Exception call writer close method
This closes #2245
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3ea2a1dd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3ea2a1dd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3ea2a1dd
Branch: refs/heads/master
Commit: 3ea2a1dd52f907fcf44966053a1a06a640654aae
Parents: 5d55b83
Author: BJangir <ba...@gmail.com>
Authored: Sat Apr 28 05:45:04 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Mon Apr 30 18:09:19 2018 +0530
----------------------------------------------------------------------
.../carbondata/sdk/file/AvroCarbonWriter.java | 13 ++++---
.../carbondata/sdk/file/CSVCarbonWriter.java | 5 ++-
.../sdk/file/CarbonWriterBuilder.java | 20 ++++++----
.../sdk/file/CSVCarbonWriterTest.java | 41 ++++++++++++++++++++
4 files changed, 64 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
index e88164c..046a4ee 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
@@ -99,14 +99,15 @@ class AvroCarbonWriter extends CarbonWriter {
*/
@Override
public void write(Object object) throws IOException {
- GenericData.Record record = (GenericData.Record) object;
-
- // convert Avro record to CSV String[]
- String[] csvRecord = avroToCsv(record);
- writable.set(csvRecord);
try {
+ GenericData.Record record = (GenericData.Record) object;
+
+ // convert Avro record to CSV String[]
+ String[] csvRecord = avroToCsv(record);
+ writable.set(csvRecord);
recordWriter.write(NullWritable.get(), writable);
- } catch (InterruptedException e) {
+ } catch (Exception e) {
+ close();
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
index df6afc6..62d5860 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
@@ -65,10 +65,11 @@ class CSVCarbonWriter extends CarbonWriter {
*/
@Override
public void write(Object object) throws IOException {
- writable.set((String[]) object);
try {
+ writable.set((String[]) object);
recordWriter.write(NullWritable.get(), writable);
- } catch (InterruptedException e) {
+ } catch (Exception e) {
+ close();
throw new IOException(e);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 5f5ee6f..770c6b0 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -334,11 +334,15 @@ public class CarbonWriterBuilder {
// If sort columns are not specified, default set all dimensions to sort column.
// When dimensions are default set to sort column,
// Inverted index will be supported by default for sort columns.
+ //Null check for field to handle hole in field[] ex.
+ // user passed size 4 but supplied only 2 fileds
for (Field field : schema.getFields()) {
- if (field.getDataType() == DataTypes.STRING ||
- field.getDataType() == DataTypes.DATE ||
- field.getDataType() == DataTypes.TIMESTAMP) {
- sortColumnsList.add(field.getFieldName());
+ if (null != field) {
+ if (field.getDataType() == DataTypes.STRING ||
+ field.getDataType() == DataTypes.DATE ||
+ field.getDataType() == DataTypes.TIMESTAMP) {
+ sortColumnsList.add(field.getFieldName());
+ }
}
}
sortColumns = new String[sortColumnsList.size()];
@@ -347,9 +351,11 @@ public class CarbonWriterBuilder {
sortColumnsList = Arrays.asList(sortColumns);
}
for (Field field : schema.getFields()) {
- tableSchemaBuilder.addColumn(
- new StructField(field.getFieldName(), field.getDataType()),
- sortColumnsList.contains(field.getFieldName()));
+ if (null != field) {
+ tableSchemaBuilder.addColumn(
+ new StructField(field.getFieldName(), field.getDataType()),
+ sortColumnsList.contains(field.getFieldName()));
+ }
}
String tableName;
String dbName;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/3ea2a1dd/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
index 41fde66..76778e5 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
@@ -205,4 +205,45 @@ public class CSVCarbonWriterTest {
FileUtils.deleteDirectory(new File(path));
}
+
+ @Test(expected = IOException.class)
+ public void testWhenWriterthrowsError() throws IOException{
+ CarbonWriter carbonWriter = null;
+ String path = "./testWriteFiles";
+
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ try {
+ carbonWriter = CarbonWriter.builder().isTransactionalTable(false).
+ outputPath(path).withSchema(new Schema(fields)).buildWriterForCSVInput();
+ } catch (InvalidLoadOptionException e) {
+ e.printStackTrace();
+ Assert.assertTrue(false);
+ }
+ carbonWriter.write("babu,1");
+ carbonWriter.close();
+
+ }
+ @Test
+ public void testWrongSchemaFieldsValidation() throws IOException{
+ CarbonWriter carbonWriter = null;
+ String path = "./testWriteFiles";
+
+ FileUtils.deleteDirectory(new File(path));
+ Field[] fields = new Field[3]; // supply 3 size fields but actual Field array value given is 2
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+ try {
+ carbonWriter = CarbonWriter.builder().isTransactionalTable(false).
+ outputPath(path).withSchema(new Schema(fields)).buildWriterForCSVInput();
+ } catch (InvalidLoadOptionException e) {
+ e.printStackTrace();
+ Assert.assertTrue(false);
+ }
+ carbonWriter.write(new String[]{"babu","1"});
+ carbonWriter.close();
+
+ }
}