You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/03/15 06:06:12 UTC
[1/2] incubator-carbondata git commit: bad record fail action
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 088240ef4 -> 68edab667
bad record fail action
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9e11e13c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9e11e13c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9e11e13c
Branch: refs/heads/master
Commit: 9e11e13c52eab3959c6bc1b4dafb587fbbb85965
Parents: 088240e
Author: akash <ak...@huawei.com>
Authored: Wed Mar 8 20:42:48 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 11:35:04 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +
.../src/test/resources/badrecords/test2.csv | 4 +
.../BadRecordLoggerSharedDictionaryTest.scala | 96 ++++++++++++++++++++
.../processing/constants/LoggerAction.java | 4 +-
.../processing/csvload/DataGraphExecuter.java | 11 ++-
.../converter/impl/RowConverterImpl.java | 4 +
.../sort/impl/ParallelReadMergeSorterImpl.java | 29 +++++-
.../newflow/sort/impl/ThreadStatusObserver.java | 40 ++++++++
.../steps/DataConverterProcessorStepImpl.java | 6 +-
...ConverterProcessorWithBucketingStepImpl.java | 6 +-
.../csvbased/BadRecordsLogger.java | 11 ++-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 46 +++++++++-
12 files changed, 247 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8d1f2d0..3eb91b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -250,6 +250,10 @@ public final class CarbonCommonConstants {
*/
public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
/**
+ * BAD_RECORD_KEY_VALUE
+ */
+ public static final String BAD_RECORD_KEY = "BADRECORD";
+ /**
* MERGERD_EXTENSION
*/
public static final String MERGERD_EXTENSION = ".merge";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/integration/spark/src/test/resources/badrecords/test2.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/badrecords/test2.csv b/integration/spark/src/test/resources/badrecords/test2.csv
new file mode 100644
index 0000000..51d25b2
--- /dev/null
+++ b/integration/spark/src/test/resources/badrecords/test2.csv
@@ -0,0 +1,4 @@
+0,569,silo
+1,843658743265874365874365874365584376547598375987,hello
+2,87436587349436587568784658743065874376,priyal
+3,56985,simple
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
new file mode 100644
index 0000000..885a0f0
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.badrecordloger
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.hive.HiveContext
+import org.scalatest.BeforeAndAfterAll
+
+
+/**
+ * Test Class for detailed query on timestamp datatypes
+ *
+ *
+ */
+class BadRecordLoggerSharedDictionaryTest extends QueryTest with BeforeAndAfterAll {
+ var hiveContext: HiveContext = _
+ var csvFilePath : String = null
+ var timestamp_format: String = null
+
+ override def beforeAll {
+ sql("drop table IF EXISTS testdrive")
+ sql(
+ """create table testdrive (ID int,CUST_ID int,cust_name string)
+ STORED BY 'org.apache.carbondata.format'
+ TBLPROPERTIES("columnproperties.cust_name.shared_column"="shared.cust_name",
+ "columnproperties.ID.shared_column"="shared.ID",
+ "columnproperties.CUST_ID.shared_column"="shared.CUST_ID",
+ 'DICTIONARY_INCLUDE'='ID,CUST_ID')"""
+ )
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ new File("./target/test/badRecords")
+ .getCanonicalPath
+ )
+
+ val carbonProp = CarbonProperties.getInstance()
+ timestamp_format = carbonProp.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ carbonProp.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+ val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+ .getCanonicalPath
+ csvFilePath = currentDirectory + "/src/test/resources/badrecords/test2.csv"
+
+ }
+ test("dataload with bad record test") {
+ try {
+ sql(
+ s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
+ |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
+ |'FILEHEADER'= 'ID,CUST_ID,cust_name', 'USE_KETTLE' = 'TRUE')""".stripMargin)
+ } catch {
+ case e: Throwable =>
+ assert(e.getMessage.contains("Data load failed due to bad record"))
+ }
+ }
+
+ test("dataload with no kettle") {
+ try {
+ sql(
+ s"""LOAD DATA INPATH '$csvFilePath' INTO TABLE testdrive OPTIONS('DELIMITER'=',',
+ |'QUOTECHAR'= '"', 'BAD_RECORDS_LOGGER_ENABLE'='TRUE', 'BAD_RECORDS_ACTION'='FAIL',
+ |'FILEHEADER'= 'ID,CUST_ID,cust_name')""".stripMargin)
+ } catch {
+ case e: Throwable =>
+ assert(e.getMessage.contains("Data load failed due to bad record"))
+ }
+ }
+
+ override def afterAll {
+ sql("drop table IF EXISTS testdrive")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timestamp_format)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java b/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
index bca9b22..901df3b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/constants/LoggerAction.java
@@ -24,8 +24,8 @@ public enum LoggerAction {
FORCE("FORCE"), // data will be converted to null
REDIRECT("REDIRECT"), // no null conversion moved to bad record and written to raw csv
- IGNORE("IGNORE"); // no null conversion moved to bad record and not written to raw csv
-
+ IGNORE("IGNORE"), // no null conversion moved to bad record and not written to raw csv
+ FAIL("FAIL"); //data loading will fail if a bad record is found
private String name;
LoggerAction(String name) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 8670182..7e6f6f4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -145,6 +145,7 @@ public class DataGraphExecuter {
model.getCsvFilePath() != null && model.getCsvFilePath().startsWith("hdfs:");
trans.setVariable("modifiedDimNames", model.getDimTables());
trans.setVariable("csvInputFilePath", model.getCsvFilePath());
+ trans.setVariable(CarbonCommonConstants.BAD_RECORD_KEY, null);
if (hdfsReadMode) {
trans.addParameterDefinition("vfs.hdfs.dfs.client.read.shortcircuit", "true", "");
trans.addParameterDefinition("vfs.hdfs.dfs.domain.socket.path",
@@ -175,12 +176,18 @@ public class DataGraphExecuter {
String key = model.getDatabaseName() + '/' + model.getTableName() + '_' + model.getTableName();
if (trans.getErrors() > 0) {
+ if (null != trans.getVariable(CarbonCommonConstants.BAD_RECORD_KEY)) {
+ LOGGER.error(trans.getVariable(CarbonCommonConstants.BAD_RECORD_KEY));
+ throw new DataLoadingException(
+ "Data load failed due to bad record ," + trans
+ .getVariable(CarbonCommonConstants.BAD_RECORD_KEY));
+ }
LOGGER.error("Graph Execution had errors");
throw new DataLoadingException("Due to internal errors, please check logs for more details.");
} else if (null != BadRecordsLogger.hasBadRecord(key)) {
- LOGGER.error("Graph Execution is partcially success");
+ LOGGER.error("Data load is partially success");
throw new DataLoadingException(DataProcessorConstants.BAD_REC_FOUND,
- "Graph Execution is partcially success");
+ "Data load is partially success");
} else {
LOGGER.info("Graph execution task is over with No error.");
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
index cdc69eb..42c8c7f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
@@ -149,6 +149,10 @@ public class RowConverterImpl implements RowConverter {
for (int i = 0; i < fieldConverters.length; i++) {
fieldConverters[i].convert(row, logHolder);
if (logHolder.isBadRecordNotAdded()) {
+ if (badRecordLogger.isDataLoadFail()) {
+ String error = "Data load failed due to bad bad record: " + logHolder.getReason();
+ throw new CarbonDataLoadingException(error);
+ }
badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
logHolder.clear();
if (badRecordLogger.isBadRecordConvertNullDisable()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
index 8c04f8f..433f8db 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
@@ -58,6 +58,8 @@ public class ParallelReadMergeSorterImpl implements Sorter {
private ExecutorService executorService;
+ private ThreadStatusObserver threadStatusObserver;
+
private SingleThreadFinalSortFilesMerger finalMerger;
private AtomicLong rowCounter;
@@ -97,18 +99,22 @@ public class ParallelReadMergeSorterImpl implements Sorter {
throw new CarbonDataLoadingException(e);
}
this.executorService = Executors.newFixedThreadPool(iterators.length);
+ this.threadStatusObserver = new ThreadStatusObserver(executorService);
try {
for (int i = 0; i < iterators.length; i++) {
executorService.submit(
- new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter));
+ new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter,
+ threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRow, sortParameters);
} catch (Exception e) {
+ checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
+ checkError();
try {
intermediateFileMerger.finish();
intermediateFileMerger = null;
@@ -148,6 +154,18 @@ public class ParallelReadMergeSorterImpl implements Sorter {
}
/**
+ * Below method will be used to check error in exception
+ */
+ private void checkError() {
+ if (threadStatusObserver.getThrowable() != null) {
+ if (threadStatusObserver.getThrowable() instanceof CarbonDataLoadingException) {
+ throw (CarbonDataLoadingException) threadStatusObserver.getThrowable();
+ } else {
+ throw new CarbonDataLoadingException(threadStatusObserver.getThrowable());
+ }
+ }
+ }
+ /**
* Below method will be used to process data to next step
*/
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters)
@@ -190,12 +208,16 @@ public class ParallelReadMergeSorterImpl implements Sorter {
private AtomicLong rowCounter;
- public SortIteratorThread(Iterator<CarbonRowBatch> iterator,
- SortDataRows sortDataRows, int batchSize, AtomicLong rowCounter) {
+ private ThreadStatusObserver observer;
+
+ public SortIteratorThread(Iterator<CarbonRowBatch> iterator, SortDataRows sortDataRows,
+ int batchSize, AtomicLong rowCounter, ThreadStatusObserver observer) {
this.iterator = iterator;
this.sortDataRows = sortDataRows;
this.buffer = new Object[batchSize][];
this.rowCounter = rowCounter;
+ this.observer = observer;
+
}
@Override
@@ -217,6 +239,7 @@ public class ParallelReadMergeSorterImpl implements Sorter {
}
} catch (Exception e) {
LOGGER.error(e);
+ observer.notifyFailed(e);
throw new CarbonDataLoadingException(e);
}
return null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
new file mode 100644
index 0000000..d901ba4
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ThreadStatusObserver.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.newflow.sort.impl;
+
+import java.util.concurrent.ExecutorService;
+
+public class ThreadStatusObserver {
+
+ private ExecutorService executorService;
+
+ private Throwable throwable;
+
+ public ThreadStatusObserver(ExecutorService executorService) {
+ this.executorService = executorService;
+ }
+
+ public void notifyFailed(Throwable throwable) {
+ executorService.shutdownNow();
+ this.throwable = throwable;
+ }
+
+ public Throwable getThrowable() {
+ return throwable;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index b900a14..c0a1cad 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -114,6 +114,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
private BadRecordsLogger createBadRecordLogger() {
boolean badRecordsLogRedirect = false;
boolean badRecordConvertNullDisable = false;
+ boolean isDataLoadFail = false;
boolean badRecordsLoggerEnable = Boolean.parseBoolean(
configuration.getDataLoadProperty(DataLoadProcessorConstants.BAD_RECORDS_LOGGER_ENABLE)
.toString());
@@ -139,6 +140,9 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
badRecordsLogRedirect = false;
badRecordConvertNullDisable = true;
break;
+ case FAIL:
+ isDataLoadFail = true;
+ break;
}
}
CarbonTableIdentifier identifier =
@@ -147,7 +151,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + File.separator + identifier.getTableName() + File.separator
+ configuration.getTaskNo()), badRecordsLogRedirect, badRecordsLoggerEnable,
- badRecordConvertNullDisable);
+ badRecordConvertNullDisable, isDataLoadFail);
return badRecordsLogger;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index db8cf42..0277138 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -141,6 +141,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
private BadRecordsLogger createBadRecordLogger() {
boolean badRecordsLogRedirect = false;
boolean badRecordConvertNullDisable = false;
+ boolean isDataLoadFail = false;
boolean badRecordsLoggerEnable = Boolean.parseBoolean(
configuration.getDataLoadProperty(DataLoadProcessorConstants.BAD_RECORDS_LOGGER_ENABLE)
.toString());
@@ -166,6 +167,9 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
badRecordsLogRedirect = false;
badRecordConvertNullDisable = true;
break;
+ case FAIL:
+ isDataLoadFail = true;
+ break;
}
}
CarbonTableIdentifier identifier =
@@ -174,7 +178,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + File.separator + identifier.getTableName() + File.separator
+ configuration.getTaskNo()), badRecordsLogRedirect, badRecordsLoggerEnable,
- badRecordConvertNullDisable);
+ badRecordConvertNullDisable, isDataLoadFail);
return badRecordsLogger;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
index ec9fbb9..3b3fd9f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
@@ -85,11 +85,13 @@ public class BadRecordsLogger {
private boolean badRecordConvertNullDisable;
+ private boolean isDataLoadFail;
+
// private final Object syncObject =new Object();
public BadRecordsLogger(String key, String fileName, String storePath,
boolean badRecordsLogRedirect, boolean badRecordLoggerEnable,
- boolean badRecordConvertNullDisable) {
+ boolean badRecordConvertNullDisable, boolean isDataLoadFail) {
// Initially no bad rec
taskKey = key;
this.fileName = fileName;
@@ -97,10 +99,11 @@ public class BadRecordsLogger {
this.badRecordsLogRedirect = badRecordsLogRedirect;
this.badRecordLoggerEnable = badRecordLoggerEnable;
this.badRecordConvertNullDisable = badRecordConvertNullDisable;
+ this.isDataLoadFail = isDataLoadFail;
}
/**
- * @param key DatabaseName/TableName/tablename
+ * @param key DatabaseNaame/TableName/tablename
* @return return "Partially" and remove from map
*/
public static String hasBadRecord(String key) {
@@ -236,6 +239,10 @@ public class BadRecordsLogger {
return badRecordConvertNullDisable;
}
+ public boolean isDataLoadFail() {
+ return isDataLoadFail;
+ }
+
/**
* closeStreams void
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9e11e13c/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 3edd43c..4faeee9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -267,6 +267,9 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
* dimension column ids
*/
private String[] dimensionColumnIds;
+
+ private Trans dis;
+
/**
* Constructor
*
@@ -280,6 +283,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
TransMeta t, Trans dis) {
super(s, stepDataInterface, c, t, dis);
csvFilepath = dis.getVariable("csvInputFilePath");
+ this.dis = dis;
+
}
/**
@@ -443,6 +448,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
boolean badRecordsLoggerEnable;
boolean badRecordsLogRedirect = false;
boolean badRecordConvertNullDisable = false;
+ boolean isDataLoadFail = false;
badRecordsLoggerEnable = Boolean
.parseBoolean(meta.getTableOptionWrapper().get(BAD_RECORDS_LOGGER_ENABLE.getName()));
String bad_records_action =
@@ -466,13 +472,17 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
badRecordsLogRedirect = false;
badRecordConvertNullDisable = true;
break;
+ case FAIL:
+ isDataLoadFail = true;
+ break;
}
}
String key = meta.getDatabaseName() + '/' + meta.getTableName() +
'_' + meta.getTableName();
badRecordsLogger = new BadRecordsLogger(key, csvFilepath, getBadLogStoreLocation(
meta.getDatabaseName() + '/' + meta.getTableName() + "/" + meta.getTaskNo()),
- badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable);
+ badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable,
+ isDataLoadFail);
HashMap<String, String> dateformatsHashMap = new HashMap<String, String>();
if (meta.dateFormat != null) {
String[] dateformats = meta.dateFormat.split(CarbonCommonConstants.COMMA);
@@ -931,6 +941,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
}
return out;
+ } catch (KettleException e) {
+ throw new RuntimeException(e);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -989,6 +1001,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
isStringDataType[j], byteBufferArr);
if (!isSerialized && !isStringDataType[j] && CarbonCommonConstants.MEMBER_DEFAULT_VAL
.equals(dimensionValue)) {
+ failDataLoad(r, index, columnName, msrDataType[meta.msrMapping[msrCount]].name());
addEntryToBadRecords(r, j, columnName, dataTypes[j]);
if (badRecordsLogger.isBadRecordConvertNullDisable()) {
return null;
@@ -1024,6 +1037,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
out[memberMapping[dimLen + index]] = surrogate.doubleValue();
} else if (!isSerialized && (isNull || msr == null
|| msr.length() == 0)) {
+ failDataLoad(r, index, columnName,
+ msrDataType[meta.msrMapping[msrCount]].name());
addEntryToBadRecords(r, j, columnName,
msrDataType[meta.msrMapping[msrCount]].name());
if (badRecordsLogger.isBadRecordConvertNullDisable()) {
@@ -1048,6 +1063,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
measureValueBasedOnDataType;
}
} catch (NumberFormatException e) {
+ failDataLoad(r, index, columnName,
+ msrDataType[meta.msrMapping[msrCount]].name());
addEntryToBadRecords(r, j, columnName, msrDataType[meta.msrMapping[msrCount]].name());
if (badRecordsLogger.isBadRecordConvertNullDisable()) {
return null;
@@ -1209,8 +1226,8 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
surrogateKeyForHrrchy[0] =
directDictionaryGenerators[m].generateDirectSurrogateKey(tuple);
if (!isSerialized && surrogateKeyForHrrchy[0] == 1) {
- addEntryToBadRecords(r, j, columnName,
- details.getColumnType().name());
+ failDataLoad(r, index, columnName, details.getColumnType().getName());
+ addEntryToBadRecords(r, j, columnName, details.getColumnType().name());
if (badRecordsLogger.isBadRecordConvertNullDisable()) {
return null;
}
@@ -1231,6 +1248,12 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
if (surrogateKeyForHrrchy[0] == CarbonCommonConstants.INVALID_SURROGATE_KEY) {
if (!isSerialized) {
+ int m = j;
+ if (isPresentInSchema) {
+ m = presentColumnMapIndex[j];
+ }
+ ColumnSchemaDetails details = columnSchemaDetailsWrapper.get(dimensionColumnIds[m]);
+ failDataLoad(r, index, columnName, details.getColumnType().getName());
addEntryToBadRecords(r, j, columnName);
if (badRecordsLogger.isBadRecordConvertNullDisable()) {
return null;
@@ -1262,6 +1285,16 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
return newArray;
}
+ private void failDataLoad(Object[] row, int index, String columnName, String dataType)
+ throws KettleException {
+ if (badRecordsLogger.isDataLoadFail()) {
+ String errorMessage = getBadRecordEntry(row, index, columnName, dataType);
+ dis.setVariable(CarbonCommonConstants.BAD_RECORD_KEY, errorMessage);
+ LOGGER.error("Data load failed due to bad record. " + errorMessage);
+ throw new KettleException("Data load failed due to bad record");
+ }
+ }
+
private void addEntryToBadRecords(Object[] r, int j, String columnName, String dataType) {
dataType = DataTypeUtil.getColumnDataTypeDisplayName(dataType);
badRecordsLogger.addBadRecordsToBuilder(r,
@@ -1269,6 +1302,13 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
+ " and column data type " + dataType + " is not a valid " + dataType + " type.");
}
+ private String getBadRecordEntry(Object[] r, int j, String columnName, String dataType) {
+ dataType = DataTypeUtil.getColumnDataTypeDisplayName(dataType);
+ String badRecord = "The value " + " \"" + r[j] + "\"" + " with column name " + columnName
+ + " and column data type " + dataType + " is not a valid Record";
+ return badRecord;
+ }
+
private void addEntryToBadRecords(Object[] r, int j, String columnName) {
badRecordsLogger.addBadRecordsToBuilder(r,
"Surrogate key for value " + " \"" + r[j] + "\"" + " with column name " + columnName
[2/2] incubator-carbondata git commit: [CARBONDATA-424] Data Load
will fail for badrecord when bad_records_action is FAIL This closes #332
Posted by ra...@apache.org.
[CARBONDATA-424] Data Load will fail for badrecord when bad_records_action is FAIL This closes #332
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/68edab66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/68edab66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/68edab66
Branch: refs/heads/master
Commit: 68edab667a7475a841e2f30cf9622d803ad9c0dd
Parents: 088240e 9e11e13
Author: ravipesala <ra...@gmail.com>
Authored: Wed Mar 15 11:35:52 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Mar 15 11:35:52 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +
.../src/test/resources/badrecords/test2.csv | 4 +
.../BadRecordLoggerSharedDictionaryTest.scala | 96 ++++++++++++++++++++
.../processing/constants/LoggerAction.java | 4 +-
.../processing/csvload/DataGraphExecuter.java | 11 ++-
.../converter/impl/RowConverterImpl.java | 4 +
.../sort/impl/ParallelReadMergeSorterImpl.java | 29 +++++-
.../newflow/sort/impl/ThreadStatusObserver.java | 40 ++++++++
.../steps/DataConverterProcessorStepImpl.java | 6 +-
...ConverterProcessorWithBucketingStepImpl.java | 6 +-
.../csvbased/BadRecordsLogger.java | 11 ++-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 46 +++++++++-
12 files changed, 247 insertions(+), 14 deletions(-)
----------------------------------------------------------------------