[carbondata] branch master updated: [HOTFIX] Fix json to carbon writer

[ku...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 1d0754e  [HOTFIX] Fix json to carbon writer
1d0754e is described below

commit 1d0754edd5af2a5abf9b7c6d7dd014e2fd74802a
Author: Jacky Li <ja...@qq.com>
AuthorDate: Thu Jul 11 15:33:12 2019 +0800

    [HOTFIX] Fix json to carbon writer
    
    When using SDK to write carbon files with json input, there is a NullPointerException.
    
    This closes #3323
---
 .../carbondata/sdk/file/CarbonWriterBuilder.java   |  9 ++--
 .../carbondata/sdk/file/CarbonReaderTest.java      | 53 ++++++++++++++++++++++
 2 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index d29ccf7..29ef81f 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -65,7 +65,10 @@ public class CarbonWriterBuilder {
   private int pageSizeInMb;
   private int blockSize;
   private long timestamp;
-  private Map<String, String> options;
+
+  // use TreeMap as keys need to be case insensitive
+  private Map<String, String> options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
   private String taskNo;
   private int localDictionaryThreshold;
   private boolean isLocalDictionaryEnabled = Boolean.parseBoolean(
@@ -239,10 +242,6 @@ public class CarbonWriterBuilder {
       }
     }
 
-    if (this.options == null) {
-      // convert it to treeMap as keys need to be case insensitive
-      this.options = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-    }
     this.options.putAll(options);
     return this;
   }
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index 9aee152..a8eb425 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -106,6 +106,59 @@ public class CarbonReaderTest extends TestCase {
     FileUtils.deleteDirectory(new File(path));
   }
 
+  @Test
+  public void testWriteAndReadJson() throws IOException, InterruptedException {
+    int numRows = 100;
+    String path = "./testWriteFiles";
+    FileUtils.deleteDirectory(new File(path));
+
+    String json = "{\"name\":\"bob\", \"age\":10}";
+
+    Schema schema = new Schema(
+        new Field[]{
+            new Field("name", "string"),
+            new Field("age", "int")});
+
+    try {
+      CarbonWriter writer = CarbonWriter.builder().outputPath(path)
+          .withJsonInput(schema).writtenBy("AvroCarbonWriterTest").build();
+
+      for (int i = 0; i < numRows; i++) {
+        writer.write(json);
+      }
+      writer.close();
+    } catch (Exception e) {
+      e.printStackTrace();
+      Assert.fail(e.getMessage());
+    }
+
+    File[] dataFiles = new File(path).listFiles(new FileFilter() {
+      @Override public boolean accept(File pathname) {
+        return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+      }
+    });
+    Assert.assertNotNull(dataFiles);
+    Assert.assertEquals(1, dataFiles.length);
+
+    // read it and verify
+
+    CarbonReader reader = CarbonReader.builder(path, "_temp")
+        .projection(new String[]{"name", "age"}).build();
+
+    int i = 0;
+    while (reader.hasNext()) {
+      Object[] row = (Object[]) reader.readNextRow();
+      Assert.assertEquals("bob", row[0]);
+      Assert.assertEquals(10, row[1]);
+      i++;
+    }
+    Assert.assertEquals(i, numRows);
+
+    reader.close();
+
+    FileUtils.deleteDirectory(new File(path));
+  }
+
   @Test public void testReadWithZeroBatchSize() throws Exception {
     String path = "./testWriteFiles";
     FileUtils.deleteDirectory(new File(path));