You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/03/15 06:06:12 UTC

[1/2] incubator-carbondata git commit: bad record fail action

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 088240ef4 -> 68edab667


bad record fail action


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9e11e13c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9e11e13c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9e11e13c

Branch: refs/heads/master
Commit: 9e11e13c52eab3959c6bc1b4dafb587fbbb85965
Parents: 088240e
Author: akash <ak...@huawei.com>
Authored: Wed Mar 8 20:42:48 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 11:35:04 2017 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  4 +
 .../src/test/resources/badrecords/test2.csv     |  4 +
 .../BadRecordLoggerSharedDictionaryTest.scala   | 96 ++++++++++++++++++++
 .../processing/constants/LoggerAction.java      |  4 +-
 .../processing/csvload/DataGraphExecuter.java   | 11 ++-
 .../converter/impl/RowConverterImpl.java        |  4 +
 .../sort/impl/ParallelReadMergeSorterImpl.java  | 29 +++++-
 .../newflow/sort/impl/ThreadStatusObserver.java | 40 ++++++++
 .../steps/DataConverterProcessorStepImpl.java   |  6 +-
 ...ConverterProcessorWithBucketingStepImpl.java |  6 +-
 .../csvbased/BadRecordsLogger.java              | 11 ++-
 .../csvbased/CarbonCSVBasedSeqGenStep.java      | 46 +++++++++-
 12 files changed, 247 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8d1f2d0..3eb91b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -250,6 +250,10 @@ public final class CarbonCommonConstants {
    */
   public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
   /**
+   * BAD_RECORD_KEY_VALUE
+   */
+  public static final String BAD_RECORD_KEY = "BADRECORD";
+  /**
    * MERGERD_EXTENSION
    */
   public static final String MERGERD_EXTENSION = ".merge";

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/integration/spark/src/test/resources/badrecords/test2.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/badrecords/test2.csv b/integration/spark/src/test/resources/badrecords/test2.csv
new file mode 100644
index 0000000..51d25b2
--- /dev/null
+++ b/integration/spark/src/test/resources/badrecords/test2.csv
@@ -0,0 +1,4 @@
+0,569,silo
+1,843658743265874365874365874365584376547598375987,hello
+2,87436587349436587568784658743065874376,priyal
+3,56985,simple
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
new file mode 100644
index 0000000..885a0f0
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.badrecordloger
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.hive.HiveContext
+import org.scalatest.BeforeAndAfterAll
+
+
+/**
+ * Test Class for detailed query on timestamp datatypes
+ *
+ *
+ */
+class BadRecordLoggerSharedDictionaryTest extends QueryTest with BeforeAndAfterAll {
+  var hiveContext: HiveContext = _
+  var csvFilePath : String = null
+  var timestamp_format: String = null
+
+  override def beforeAll {
+      sql("drop table IF EXISTS testdrive")
+    sql(
+      """create table testdrive (ID int,CUST_ID int,cust_name string)
+          STORED BY 'org.apache.carbondata.format'
+            TBLPROPERTIES("columnproperties.cust_name.shared_column"="shared.cust_name",
+            "columnproperties.ID.shared_column"="shared.ID",
+            "columnproperties.CUST_ID.shared_column"="shared.CUST_ID",
+            'DICTIONARY_INCLUDE'='ID,CUST_ID')"""
+    )
+
+      CarbonProperties.getInstance()
+        .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+          new File("./target/test/badRecords")
+            .getCanonicalPath
+        )
+
+    val carbonProp = CarbonProperties.getInstance()
+    timestamp_format = carbonProp.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+      CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+    carbonProp.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    csvFilePath = currentDirectory + "/src/test/resources/badrecords/test2.csv"
+
+  }
+  test("dataload with bad record test") {
+    try {
+      sql(
+        s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
+            |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
+            |'FILEHEADER'= 'ID,CUST_ID,cust_name', 'USE_KETTLE' = 'TRUE')""".stripMargin)
+    } catch {
+      case e: Throwable =>
+        assert(e.getMessage.contains("Data load failed due to bad record"))
+    }
+  }
+
+  test("dataload with no kettle") {
+    try {
+      sql(
+        s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
+            |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
+            |'FILEHEADER'= 'ID,CUST_ID,cust_name')""".stripMargin)
+    } catch {
+      case e: Throwable =>
+        assert(e.getMessage.contains("Data load failed due to bad record"))
+    }
+  }
+
+  override def afterAll {
+    sql("drop table IF EXISTS testdrive")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestamp_format)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java b/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
index bca9b22..901df3b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
@@ -24,8 +24,8 @@ public enum LoggerAction {
 
   FORCE("FORCE"), // data will be converted to null
   REDIRECT("REDIRECT"), // no null conversion moved to bad record and written to raw csv
-  IGNORE("IGNORE"); // no null conversion moved to bad record and not written to raw csv
-
+  IGNORE("IGNORE"), // no null conversion moved to bad record and not written to raw csv
+  FAIL("FAIL");  //data loading will fail if a bad record is found
   private String name;
 
   LoggerAction(String name) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 8670182..7e6f6f4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -145,6 +145,7 @@ public class DataGraphExecuter {
             model.getCsvFilePath() != null && model.getCsvFilePath().startsWith("hdfs:");
         trans.setVariable("modifiedDimNames", model.getDimTables());
         trans.setVariable("csvInputFilePath", model.getCsvFilePath());
+        trans.setVariable(CarbonCommonConstants.BAD_RECORD_KEY, null);
         if (hdfsReadMode) {
           trans.addParameterDefinition("vfs.hdfs.dfs.client.read.shortcircuit", "true", "");
           trans.addParameterDefinition("vfs.hdfs.dfs.domain.socket.path",
@@ -175,12 +176,18 @@ public class DataGraphExecuter {
     String key = model.getDatabaseName() + '/' + model.getTableName() + '_' + model.getTableName();
 
     if (trans.getErrors() > 0) {
+      if (null != trans.getVariable(CarbonCommonConstants.BAD_RECORD_KEY)) {
+        LOGGER.error(trans.getVariable(CarbonCommonConstants.BAD_RECORD_KEY));
+        throw new DataLoadingException(
+            "Data load failed due to bad record ," + trans
+                .getVariable(CarbonCommonConstants.BAD_RECORD_KEY));
+      }
       LOGGER.error("Graph Execution had errors");
       throw new DataLoadingException("Due to internal errors, please check logs for more details.");
     } else if (null != BadRecordsLogger.hasBadRecord(key)) {
-      LOGGER.error("Graph Execution is partcially success");
+      LOGGER.error("Data load is partially success");
       throw new DataLoadingException(DataProcessorConstants.BAD_REC_FOUND,
-          "Graph Execution is partcially success");
+          "Data load is partially success");
     } else {
       LOGGER.info("Graph execution task is over with No error.");
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
index cdc69eb..42c8c7f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
@@ -149,6 +149,10 @@ public class RowConverterImpl implements RowConverter {
     for (int i = 0; i < fieldConverters.length; i++) {
       fieldConverters[i].convert(row, logHolder);
       if (logHolder.isBadRecordNotAdded()) {
+        if (badRecordLogger.isDataLoadFail()) {
+          String error = "Data load failed due to bad bad record: " + logHolder.getReason();
+          throw new CarbonDataLoadingException(error);
+        }
         badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
         logHolder.clear();
         if (badRecordLogger.isBadRecordConvertNullDisable()) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
index 8c04f8f..433f8db 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
@@ -58,6 +58,8 @@ public class ParallelReadMergeSorterImpl implements Sorter {
 
   private ExecutorService executorService;
 
+  private ThreadStatusObserver threadStatusObserver;
+
   private SingleThreadFinalSortFilesMerger finalMerger;
 
   private AtomicLong rowCounter;
@@ -97,18 +99,22 @@ public class ParallelReadMergeSorterImpl implements Sorter {
       throw new CarbonDataLoadingException(e);
     }
     this.executorService = Executors.newFixedThreadPool(iterators.length);
+    this.threadStatusObserver = new ThreadStatusObserver(executorService);
 
     try {
       for (int i = 0; i < iterators.length; i++) {
         executorService.submit(
-            new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter));
+            new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter,
+                threadStatusObserver));
       }
       executorService.shutdown();
       executorService.awaitTermination(2, TimeUnit.DAYS);
       processRowToNextStep(sortDataRow, sortParameters);
     } catch (Exception e) {
+      checkError();
       throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
     }
+    checkError();
     try {
       intermediateFileMerger.finish();
       intermediateFileMerger = null;
@@ -148,6 +154,18 @@ public class ParallelReadMergeSorterImpl implements Sorter {
   }
 
   /**
+   * Below method will be used to check error in exception
+   */
+  private void checkError() {
+    if (threadStatusObserver.getThrowable() != null) {
+      if (threadStatusObserver.getThrowable() instanceof CarbonDataLoadingException) {
+        throw (CarbonDataLoadingException) threadStatusObserver.getThrowable();
+      } else {
+        throw new CarbonDataLoadingException(threadStatusObserver.getThrowable());
+      }
+    }
+  }
+  /**
    * Below method will be used to process data to next step
    */
   private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters)
@@ -190,12 +208,16 @@ public class ParallelReadMergeSorterImpl implements Sorter {
 
     private AtomicLong rowCounter;
 
-    public SortIteratorThread(Iterator<CarbonRowBatch> iterator,
-        SortDataRows sortDataRows, int batchSize, AtomicLong rowCounter) {
+    private ThreadStatusObserver observer;
+
+    public SortIteratorThread(Iterator<CarbonRowBatch> iterator, SortDataRows sortDataRows,
+        int batchSize, AtomicLong rowCounter, ThreadStatusObserver observer) {
       this.iterator = iterator;
       this.sortDataRows = sortDataRows;
       this.buffer = new Object[batchSize][];
       this.rowCounter = rowCounter;
+      this.observer = observer;
+
     }
 
     @Override
@@ -217,6 +239,7 @@ public class ParallelReadMergeSorterImpl implements Sorter {
         }
       } catch (Exception e) {
         LOGGER.error(e);
+        observer.notifyFailed(e);
         throw new CarbonDataLoadingException(e);
       }
       return null;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
new file mode 100644
index 0000000..d901ba4
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.newflow.sort.impl;
+
+import java.util.concurrent.ExecutorService;
+
+public class ThreadStatusObserver {
+
+  private ExecutorService executorService;
+
+  private Throwable throwable;
+
+  public ThreadStatusObserver(ExecutorService executorService) {
+    this.executorService = executorService;
+  }
+
+  public void notifyFailed(Throwable throwable) {
+    executorService.shutdownNow();
+    this.throwable = throwable;
+  }
+
+  public Throwable getThrowable() {
+    return throwable;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index b900a14..c0a1cad 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -114,6 +114,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
   private BadRecordsLogger createBadRecordLogger() {
     boolean badRecordsLogRedirect = false;
     boolean badRecordConvertNullDisable = false;
+    boolean isDataLoadFail = false;
     boolean badRecordsLoggerEnable = Boolean.parseBoolean(
         configuration.getDataLoadProperty(DataLoadProcessorConstants.BAD_RECORDS_LOGGER_ENABLE)
             .toString());
@@ -139,6 +140,9 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
           badRecordsLogRedirect = false;
           badRecordConvertNullDisable = true;
           break;
+        case FAIL:
+          isDataLoadFail = true;
+          break;
       }
     }
     CarbonTableIdentifier identifier =
@@ -147,7 +151,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
         identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
         identifier.getDatabaseName() + File.separator + identifier.getTableName() + File.separator
             + configuration.getTaskNo()), badRecordsLogRedirect, badRecordsLoggerEnable,
-        badRecordConvertNullDisable);
+        badRecordConvertNullDisable, isDataLoadFail);
     return badRecordsLogger;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index db8cf42..0277138 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -141,6 +141,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
   private BadRecordsLogger createBadRecordLogger() {
     boolean badRecordsLogRedirect = false;
     boolean badRecordConvertNullDisable = false;
+    boolean isDataLoadFail = false;
     boolean badRecordsLoggerEnable = Boolean.parseBoolean(
         configuration.getDataLoadProperty(DataLoadProcessorConstants.BAD_RECORDS_LOGGER_ENABLE)
             .toString());
@@ -166,6 +167,9 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
           badRecordsLogRedirect = false;
           badRecordConvertNullDisable = true;
           break;
+        case FAIL:
+          isDataLoadFail = true;
+          break;
       }
     }
     CarbonTableIdentifier identifier =
@@ -174,7 +178,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
         identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
         identifier.getDatabaseName() + File.separator + identifier.getTableName() + File.separator
             + configuration.getTaskNo()), badRecordsLogRedirect, badRecordsLoggerEnable,
-        badRecordConvertNullDisable);
+        badRecordConvertNullDisable, isDataLoadFail);
     return badRecordsLogger;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
index ec9fbb9..3b3fd9f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
@@ -85,11 +85,13 @@ public class BadRecordsLogger {
 
   private boolean badRecordConvertNullDisable;
 
+  private boolean isDataLoadFail;
+
   // private final Object syncObject =new Object();
 
   public BadRecordsLogger(String key, String fileName, String storePath,
       boolean badRecordsLogRedirect, boolean badRecordLoggerEnable,
-      boolean badRecordConvertNullDisable) {
+      boolean badRecordConvertNullDisable, boolean isDataLoadFail) {
     // Initially no bad rec
     taskKey = key;
     this.fileName = fileName;
@@ -97,10 +99,11 @@ public class BadRecordsLogger {
     this.badRecordsLogRedirect = badRecordsLogRedirect;
     this.badRecordLoggerEnable = badRecordLoggerEnable;
     this.badRecordConvertNullDisable = badRecordConvertNullDisable;
+    this.isDataLoadFail = isDataLoadFail;
   }
 
   /**
-   * @param key DatabaseName/TableName/tablename
+   * @param key DatabaseNaame/TableName/tablename
    * @return return "Partially" and remove from map
    */
   public static String hasBadRecord(String key) {
@@ -236,6 +239,10 @@ public class BadRecordsLogger {
     return badRecordConvertNullDisable;
   }
 
+  public boolean isDataLoadFail() {
+    return isDataLoadFail;
+  }
+
   /**
    * closeStreams void
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 3edd43c..4faeee9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -267,6 +267,9 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
    * dimension column ids
    */
   private String[] dimensionColumnIds;
+
+  private Trans dis;
+
   /**
    * Constructor
    *
@@ -280,6 +283,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
       TransMeta t, Trans dis) {
     super(s, stepDataInterface, c, t, dis);
     csvFilepath = dis.getVariable("csvInputFilePath");
+    this.dis = dis;
+
   }
 
   /**
@@ -443,6 +448,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
         boolean badRecordsLoggerEnable;
         boolean badRecordsLogRedirect = false;
         boolean badRecordConvertNullDisable = false;
+        boolean isDataLoadFail = false;
         badRecordsLoggerEnable = Boolean
             .parseBoolean(meta.getTableOptionWrapper().get(BAD_RECORDS_LOGGER_ENABLE.getName()));
         String bad_records_action =
@@ -466,13 +472,17 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
               badRecordsLogRedirect = false;
               badRecordConvertNullDisable = true;
               break;
+            case FAIL:
+              isDataLoadFail = true;
+              break;
           }
         }
         String key = meta.getDatabaseName() + '/' + meta.getTableName() +
             '_' + meta.getTableName();
         badRecordsLogger = new BadRecordsLogger(key, csvFilepath, getBadLogStoreLocation(
             meta.getDatabaseName() + '/' + meta.getTableName() + "/" + meta.getTaskNo()),
-            badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable);
+            badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable,
+            isDataLoadFail);
         HashMap<String, String> dateformatsHashMap = new HashMap<String, String>();
         if (meta.dateFormat != null) {
           String[] dateformats = meta.dateFormat.split(CarbonCommonConstants.COMMA);
@@ -931,6 +941,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
       }
       return out;
 
+    } catch (KettleException e) {
+      throw new RuntimeException(e);
     } catch (Exception e) {
       throw new RuntimeException(e);
     }
@@ -989,6 +1001,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
                 isStringDataType[j], byteBufferArr);
         if (!isSerialized && !isStringDataType[j] && CarbonCommonConstants.MEMBER_DEFAULT_VAL
             .equals(dimensionValue)) {
+          failDataLoad(r, index, columnName, msrDataType[meta.msrMapping[msrCount]].name());
           addEntryToBadRecords(r, j, columnName, dataTypes[j]);
           if (badRecordsLogger.isBadRecordConvertNullDisable()) {
             return null;
@@ -1024,6 +1037,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
           out[memberMapping[dimLen + index]] = surrogate.doubleValue();
         } else if (!isSerialized &&  (isNull || msr == null
             || msr.length() == 0)) {
+          failDataLoad(r, index, columnName,
+              msrDataType[meta.msrMapping[msrCount]].name());
           addEntryToBadRecords(r, j, columnName,
               msrDataType[meta.msrMapping[msrCount]].name());
           if (badRecordsLogger.isBadRecordConvertNullDisable()) {
@@ -1048,6 +1063,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
                   measureValueBasedOnDataType;
             }
           } catch (NumberFormatException e) {
+            failDataLoad(r, index, columnName,
+                msrDataType[meta.msrMapping[msrCount]].name());
             addEntryToBadRecords(r, j, columnName, msrDataType[meta.msrMapping[msrCount]].name());
             if (badRecordsLogger.isBadRecordConvertNullDisable()) {
               return null;
@@ -1209,8 +1226,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
               surrogateKeyForHrrchy[0] =
                   directDictionaryGenerators[m].generateDirectSurrogateKey(tuple);
               if (!isSerialized && surrogateKeyForHrrchy[0] == 1) {
-                addEntryToBadRecords(r, j, columnName,
-                    details.getColumnType().name());
+                failDataLoad(r, index, columnName, details.getColumnType().getName());
+                addEntryToBadRecords(r, j, columnName, details.getColumnType().name());
                 if (badRecordsLogger.isBadRecordConvertNullDisable()) {
                   return null;
                 }
@@ -1231,6 +1248,12 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
           if (surrogateKeyForHrrchy[0] == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
 
             if (!isSerialized) {
+              int m = j;
+              if (isPresentInSchema) {
+                m = presentColumnMapIndex[j];
+              }
+              ColumnSchemaDetails details = columnSchemaDetailsWrapper.get(dimensionColumnIds[m]);
+              failDataLoad(r, index, columnName, details.getColumnType().getName());
               addEntryToBadRecords(r, j, columnName);
               if (badRecordsLogger.isBadRecordConvertNullDisable()) {
                 return null;
@@ -1262,6 +1285,16 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
     return newArray;
   }
 
+  private void failDataLoad(Object[] row, int index, String columnName, String dataType)
+      throws KettleException {
+    if (badRecordsLogger.isDataLoadFail()) {
+      String errorMessage = getBadRecordEntry(row, index, columnName, dataType);
+      dis.setVariable(CarbonCommonConstants.BAD_RECORD_KEY, errorMessage);
+      LOGGER.error("Data load failed due to bad record. " + errorMessage);
+      throw new KettleException("Data load failed due to bad record");
+    }
+  }
+
   private void addEntryToBadRecords(Object[] r, int j, String columnName, String dataType) {
     dataType = DataTypeUtil.getColumnDataTypeDisplayName(dataType);
     badRecordsLogger.addBadRecordsToBuilder(r,
@@ -1269,6 +1302,13 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
             + " and column data type " + dataType + " is not a valid " + dataType + " type.");
   }
 
+  private String getBadRecordEntry(Object[] r, int j, String columnName, String dataType) {
+    dataType = DataTypeUtil.getColumnDataTypeDisplayName(dataType);
+    String badRecord = "The value " + " \"" + r[j] + "\"" + " with column name " + columnName
+        + " and column data type " + dataType + " is not a valid Record";
+    return badRecord;
+  }
+
   private void addEntryToBadRecords(Object[] r, int j, String columnName) {
     badRecordsLogger.addBadRecordsToBuilder(r,
         "Surrogate key for value " + " \"" + r[j] + "\"" + " with column name " + columnName


[2/2] incubator-carbondata git commit: [CARBONDATA-424] Data Load will fail for badrecord when bad_records_action is FAIL This closes #332

Posted by ra...@apache.org.
[CARBONDATA-424] Data Load will fail for badrecord when bad_records_action is FAIL This closes #332


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/68edab66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/68edab66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/68edab66

Branch: refs/heads/master
Commit: 68edab667a7475a841e2f30cf9622d803ad9c0dd
Parents: 088240e 9e11e13
Author: ravipesala <ra...@gmail.com>
Authored: Wed Mar 15 11:35:52 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 11:35:52 2017 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  4 +
 .../src/test/resources/badrecords/test2.csv     |  4 +
 .../BadRecordLoggerSharedDictionaryTest.scala   | 96 ++++++++++++++++++++
 .../processing/constants/LoggerAction.java      |  4 +-
 .../processing/csvload/DataGraphExecuter.java   | 11 ++-
 .../converter/impl/RowConverterImpl.java        |  4 +
 .../sort/impl/ParallelReadMergeSorterImpl.java  | 29 +++++-
 .../newflow/sort/impl/ThreadStatusObserver.java | 40 ++++++++
 .../steps/DataConverterProcessorStepImpl.java   |  6 +-
 ...ConverterProcessorWithBucketingStepImpl.java |  6 +-
 .../csvbased/BadRecordsLogger.java              | 11 ++-
 .../csvbased/CarbonCSVBasedSeqGenStep.java      | 46 +++++++++-
 12 files changed, 247 insertions(+), 14 deletions(-)
----------------------------------------------------------------------