You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/09 13:30:02 UTC
[1/2] carbondata git commit: [CARBONDATA-1049] Avoid logging raw data
into driver and executor log.// added isDebuggEnabled check
Repository: carbondata
Updated Branches:
refs/heads/master 657902d7d -> f53ab4b2b
[CARBONDATA-1049] Avoid logging raw data into driver and executor log.// added isDebuggEnabled check
[CARBONDATA-1049] Avoid logging raw data into driver and executor log.
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4a79a86a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4a79a86a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4a79a86a
Branch: refs/heads/master
Commit: 4a79a86ab06a3b053ac01094a2525a534a639d3a
Parents: 657902d
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri May 12 12:36:42 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jun 9 18:59:19 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/cache/CarbonLRUCache.java | 10 +++++---
.../DateDirectDictionaryGenerator.java | 13 ++++++----
.../TimeStampDirectDictionaryGenerator.java | 15 +++++++-----
.../core/mutate/DeleteDeltaBlockDetails.java | 4 +++-
.../core/mutate/SegmentUpdateDetails.java | 6 +++--
.../data/BlockletDeleteDeltaCacheLoader.java | 8 +++++--
.../scan/filter/FilterExpressionProcessor.java | 19 ++++++++-------
.../carbondata/core/scan/filter/FilterUtil.java | 4 +++-
.../carbondata/core/util/DataTypeUtil.java | 8 +++----
.../badrecordloger/BadRecordLoggerTest.scala | 2 +-
.../newflow/converter/BadRecordLogHolder.java | 21 ++++++++++++++++
.../impl/DictionaryFieldConverterImpl.java | 12 ++++++----
.../DirectDictionaryFieldConverterImpl.java | 18 +++++++-------
.../impl/MeasureFieldConverterImpl.java | 25 +++++++++++++-------
.../impl/NonDictionaryFieldConverterImpl.java | 12 ++++++----
.../converter/impl/RowConverterImpl.java | 8 +++++--
.../unsafe/merger/UnsafeIntermediateMerger.java | 10 +++++---
.../sortandgroupby/sortdata/SortDataRows.java | 8 +++++--
.../sortdata/SortIntermediateFileMerger.java | 4 +++-
.../store/writer/AbstractFactDataWriter.java | 6 +++--
.../util/CarbonDataProcessorUtil.java | 12 ++++++++++
21 files changed, 156 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
index 151ec58..23dd814 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CarbonLRUCache.java
@@ -171,8 +171,10 @@ public final class CarbonLRUCache {
* @param cacheInfo
*/
public boolean put(String columnIdentifier, Cacheable cacheInfo, long requiredSize) {
- LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize
- + " Current cache size :: " + currentSize);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Required size for entry " + columnIdentifier + " :: " + requiredSize
+ + " Current cache size :: " + currentSize);
+ }
boolean columnKeyAddedSuccessfully = false;
if (isLRUCacheSizeConfigured()) {
synchronized (lruCacheMap) {
@@ -206,7 +208,9 @@ public final class CarbonLRUCache {
if (null == lruCacheMap.get(columnIdentifier)) {
lruCacheMap.put(columnIdentifier, cacheInfo);
}
- LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Added entry to InMemory lru cache :: " + columnIdentifier);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index a21fd85..9fc2c25 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -109,9 +109,10 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
}
dateToStr = simpleDateFormat.parse(memberStr);
} catch (ParseException e) {
- LOGGER.debug(
- "Cannot convert " + memberStr + " to Time/Long type value. Value considered as null." + e
- .getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Cannot convert value to Time/Long type value. Value considered as null." + e
+ .getMessage());
+ }
dateToStr = null;
}
//adding +2 to reserve the first cuttOffDiff value for null or empty date
@@ -140,8 +141,10 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
try {
timeValue = Long.valueOf(memberStr) / 1000;
} catch (NumberFormatException e) {
- LOGGER.debug("Cannot convert " + memberStr + " Long type value. Value considered as null." + e
- .getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Cannot convert value to Long type value. Value considered as null." + e.getMessage());
+ }
}
if (timeValue == -1) {
return 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 7d3d6c8..5c7ba07 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -163,9 +163,10 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
}
dateToStr = simpleDateFormat.parse(memberStr);
} catch (ParseException e) {
- LOGGER.debug(
- "Cannot convert " + memberStr + " to Time/Long type value. Value considered as null." + e
- .getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Cannot convert value to Time/Long type value. Value considered as null." + e
+ .getMessage());
+ }
dateToStr = null;
}
//adding +2 to reserve the first cuttOffDiff value for null or empty date
@@ -200,9 +201,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
try {
timeValue = Long.valueOf(memberStr) / 1000;
} catch (NumberFormatException e) {
- LOGGER.debug(
- "Cannot convert " + memberStr + " Long type value. Value considered as null." + e
- .getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Cannot convert " + memberStr + " Long type value. Value considered as null." + e
+ .getMessage());
+ }
}
if (timeValue == -1) {
return 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index c4e9ea2..b1293e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -86,7 +86,9 @@ public class DeleteDeltaBlockDetails implements Serializable {
blocklet.addDeletedRow(CarbonUpdateUtil.getIntegerValue(offset));
return addBlockletDetails(blocklet);
} catch (Exception e) {
- LOGGER.debug(e.getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(e.getMessage());
+ }
throw e;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
index 6f7eef4..6f99b3a 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/SegmentUpdateDetails.java
@@ -158,8 +158,10 @@ public class SegmentUpdateDetails implements Serializable {
try {
longValue = Long.parseLong(timtstamp);
} catch (NumberFormatException nfe) {
- String errorMsg = "Invalid timestamp : " + timtstamp;
- LOGGER.debug(errorMsg);
+ if (LOGGER.isDebugEnabled()) {
+ String errorMsg = "Invalid timestamp : " + timtstamp;
+ LOGGER.debug(errorMsg);
+ }
}
return longValue;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
index 6665c5b..93607cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
@@ -57,7 +57,9 @@ public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderInt
deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
} catch (Exception e) {
- LOGGER.debug("Unable to retrieve delete delta files");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Unable to retrieve delete delta files");
+ }
}
} else {
deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
@@ -71,7 +73,9 @@ public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderInt
deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
} catch (Exception e) {
- LOGGER.debug("Unable to retrieve delete delta files");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Unable to retrieve delete delta files");
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 5e69f3a..ce31283 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -107,8 +107,10 @@ public class FilterExpressionProcessor implements FilterProcessor {
List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
// getting the start and end index key based on filter for hitting the
// selected block reference nodes based on filter resolver tree.
- LOGGER.debug("preparing the start and end key for finding"
- + "start and end block as per filter resolver");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("preparing the start and end key for finding"
+ + "start and end block as per filter resolver");
+ }
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
filterResolver, listOfStartEndKeys);
@@ -127,12 +129,13 @@ public class FilterExpressionProcessor implements FilterProcessor {
return listOfDataBlocksToScan;
}
}
-
- LOGGER.debug(
- "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
- .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
- + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
- + searchEndKey.getNoDictionaryKeys());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
+ .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
+ + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+ }
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 5434a5f..b06789e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -503,7 +503,9 @@ public final class FilterUtil {
}
}
} catch (FilterIllegalMemberException e) {
- LOGGER.debug(e.getMessage());
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(e.getMessage());
+ }
}
}
return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 596f057..af3ff2f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -287,7 +287,7 @@ public final class DataTypeUtil {
Date dateToStr = dateformatter.get().parse(data);
return dateToStr.getTime() * 1000;
} catch (ParseException e) {
- LOGGER.error("Cannot convert" + data + " to Time/Long type value" + e.getMessage());
+ LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
@@ -299,7 +299,7 @@ public final class DataTypeUtil {
Date dateToStr = timeStampformatter.get().parse(data);
return dateToStr.getTime() * 1000;
} catch (ParseException e) {
- LOGGER.error("Cannot convert" + data + " to Time/Long type value" + e.getMessage());
+ LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
case DECIMAL:
@@ -436,7 +436,7 @@ public final class DataTypeUtil {
Date dateToStr = dateformatter.get().parse(data5);
return dateToStr.getTime() * 1000;
} catch (ParseException e) {
- LOGGER.error("Cannot convert" + data5 + " to Time/Long type value" + e.getMessage());
+ LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
@@ -449,7 +449,7 @@ public final class DataTypeUtil {
Date dateToStr = timeStampformatter.get().parse(data6);
return dateToStr.getTime() * 1000;
} catch (ParseException e) {
- LOGGER.error("Cannot convert" + data6 + " to Time/Long type value" + e.getMessage());
+ LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
case DECIMAL:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
index 5716978..1c624bf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerTest.scala
@@ -193,7 +193,7 @@ class BadRecordLoggerTest extends QueryTest with BeforeAndAfterAll {
test("select count(*) from insufficientColumn") {
checkAnswer(
sql("select count(*) from insufficientColumn"),
- Seq(Row(1)
+ Seq(Row(3)
)
)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
index 42a6fec..f7ce620 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/BadRecordLogHolder.java
@@ -16,11 +16,19 @@
*/
package org.apache.carbondata.processing.newflow.converter;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* It is holder for reason of bad records.
*/
public class BadRecordLogHolder {
+ /**
+ * this map will hold the bad record unified message for columns
+ */
+ private Map<String, String> columnMessageMap = new HashMap<>();
+
private String reason;
private boolean badRecordAdded;
@@ -51,4 +59,17 @@ public class BadRecordLogHolder {
public void setLogged(boolean logged) {
isLogged = logged;
}
+
+ public Map<String, String> getColumnMessageMap() {
+ return columnMessageMap;
+ }
+
+ /**
+ * this method will clear the map entries
+ */
+ public void finish() {
+ if (null != columnMessageMap) {
+ columnMessageMap.clear();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 23e9c8c..81b44f3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.dictionary.DictionaryServerClien
import org.apache.carbondata.processing.newflow.dictionary.PreCreatedDictionary;
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConverterImpl {
@@ -106,11 +107,12 @@ public class DictionaryFieldConverterImpl extends AbstractDictionaryFieldConvert
}
if (null == parsedValue) {
if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
- String dataType = carbonDimension.getDataType().getName();
- logHolder.setReason(
- "The value " + " \"" + dimensionValue + "\"" + " with column name " + carbonDimension
- .getColName() + " and column data type " + dataType + " is not a valid "
- + dataType + " type.");
+ String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName());
+ if (null == message) {
+ message = CarbonDataProcessorUtil
+ .prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType());
+ logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message);
+ } logHolder.setReason(message);
}
row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, index);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
index 05ffe8b..39e721b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldConverterImpl {
@@ -58,10 +59,8 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
String value = row.getString(index);
if (value == null) {
- logHolder.setReason(
- "The value " + " \"" + row.getString(index) + "\"" + " with column name " + column
- .getColName() + " and column data type " + column.getDataType() + " is not a valid "
- + column.getDataType() + " type.");
+ logHolder.setReason(CarbonDataProcessorUtil
+ .prepareFailureReason(column.getColName(), column.getDataType()));
row.update(1, index);
} else if (value.equals(nullFormat)) {
row.update(1, index);
@@ -69,10 +68,13 @@ public class DirectDictionaryFieldConverterImpl extends AbstractDictionaryFieldC
int key = directDictionaryGenerator.generateDirectSurrogateKey(value);
if (key == 1) {
if ((value.length() > 0) || (value.length() == 0 && isEmptyBadRecord)) {
- logHolder.setReason(
- "The value " + " \"" + row.getString(index) + "\"" + " with column name " + column
- .getColName() + " and column data type " + column.getDataType()
- + " is not a valid " + column.getDataType() + " type.");
+ String message = logHolder.getColumnMessageMap().get(column.getColName());
+ if (null == message) {
+ message = CarbonDataProcessorUtil
+ .prepareFailureReason(column.getColName(), column.getDataType());
+ logHolder.getColumnMessageMap().put(column.getColName(), message);
+ }
+ logHolder.setReason(message);
}
}
row.update(key, index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
index bb5f983..01ea437 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.newflow.converter.FieldConverter;
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
/**
* Converter for measure
@@ -62,15 +63,22 @@ public class MeasureFieldConverterImpl implements FieldConverter {
Object output;
boolean isNull = CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(value);
if (value == null || isNull) {
- logHolder.setReason(
- "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
- + " and column data type " + dataType + " is not a valid " + dataType + " type.");
+ String message = logHolder.getColumnMessageMap().get(measure.getColName());
+ if (null == message) {
+ value = CarbonDataProcessorUtil
+ .prepareFailureReason(measure.getColName(), measure.getDataType());
+ logHolder.getColumnMessageMap().put(measure.getColName(), message);
+ }
row.update(null, index);
} else if (value.length() == 0) {
if (isEmptyBadRecord) {
- logHolder.setReason(
- "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
- + " and column data type " + dataType + " is not a valid " + dataType + " type.");
+ String message = logHolder.getColumnMessageMap().get(measure.getColName());
+ if (null == message) {
+ message = CarbonDataProcessorUtil
+ .prepareFailureReason(measure.getColName(), measure.getDataType());
+ logHolder.getColumnMessageMap().put(measure.getColName(), message);
+ }
+ logHolder.setReason(message);
}
row.update(null, index);
} else if (value.equals(nullformat)) {
@@ -81,10 +89,9 @@ public class MeasureFieldConverterImpl implements FieldConverter {
row.update(output, index);
} catch (NumberFormatException e) {
LOGGER.warn(
- "Cant not convert : " + value + " to Numeric type value. Value considered as null.");
+ "Cant not convert value to Numeric type value. Value considered as null.");
logHolder.setReason(
- "The value " + " \"" + value + "\"" + " with column name " + measure.getColName()
- + " and column data type " + dataType + " is not a valid " + dataType + " type.");
+ CarbonDataProcessorUtil.prepareFailureReason(measure.getColName(), dataType));
output = null;
row.update(output, index);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
index b39ef11..0f88d74 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.newflow.converter.FieldConverter;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class NonDictionaryFieldConverterImpl implements FieldConverter {
@@ -56,10 +57,13 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
.getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType), index);
} catch (Throwable ex) {
if (dimensionValue.length() > 0 || isEmptyBadRecord) {
- logHolder.setReason(
- "The value " + " \"" + dimensionValue + "\"" + " with column name " + column
- .getColName() + " and column data type " + dataType + " is not a valid "
- + dataType + " type.");
+ String message = logHolder.getColumnMessageMap().get(column.getColName());
+ if (null == message) {
+ message = CarbonDataProcessorUtil
+ .prepareFailureReason(column.getColName(), column.getDataType());
+ logHolder.getColumnMessageMap().put(column.getColName(), message);
+ }
+ logHolder.setReason(message);
row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
} else {
row.update(new byte[0], index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
index 4cb7a06..2414d58 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/RowConverterImpl.java
@@ -156,11 +156,12 @@ public class RowConverterImpl implements RowConverter {
for (int i = 0; i < fieldConverters.length; i++) {
fieldConverters[i].convert(row, logHolder);
if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) {
+ badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
if (badRecordLogger.isDataLoadFail()) {
- String error = "Data load failed due to bad record: " + logHolder.getReason();
+ String error = "Data load failed due to bad record: " + logHolder.getReason() +
+ "Please enable bad record logger to know the detail reason.";
throw new BadRecordFoundException(error);
}
- badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
logHolder.clear();
logHolder.setLogged(true);
if (badRecordLogger.isBadRecordConvertNullDisable()) {
@@ -180,6 +181,9 @@ public class RowConverterImpl implements RowConverter {
client.shutDown();
}
}
+ if (null != logHolder) {
+ logHolder.finish();
+ }
if (executorService != null) {
executorService.shutdownNow();
executorService = null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
index d7b7b0b..d10c75f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -93,7 +93,9 @@ public class UnsafeIntermediateMerger {
fileList = procFiles.toArray(new File[procFiles.size()]);
this.procFiles = new ArrayList<File>();
}
- LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+ }
startIntermediateMerging(fileList);
}
}
@@ -124,8 +126,10 @@ public class UnsafeIntermediateMerger {
localRowPages = rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]);
this.rowPages = new ArrayList<>();
}
- LOGGER.debug("Sumitting request for intermediate merging of in-memory pages : "
- + localRowPages.length);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Sumitting request for intermediate merging of in-memory pages : "
+ + localRowPages.length);
+ }
startIntermediateMerging(localRowPages, totalRows);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index e840984..5df3112 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -121,7 +121,9 @@ public class SortDataRows {
int currentSize = entryCount;
if (sortBufferSize == currentSize) {
- LOGGER.debug("************ Writing to temp file ********** ");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("************ Writing to temp file ********** ");
+ }
intermediateFileMerger.startMergingIfPossible();
Object[][] recordHolderListLocal = recordHolderList;
try {
@@ -151,7 +153,9 @@ public class SortDataRows {
synchronized (addRowsLock) {
int sizeLeft = 0;
if (entryCount + size >= sortBufferSize) {
- LOGGER.debug("************ Writing to temp file ********** ");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("************ Writing to temp file ********** ");
+ }
intermediateFileMerger.startMergingIfPossible();
Object[][] recordHolderListLocal = recordHolderList;
sizeLeft = sortBufferSize - entryCount ;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
index 2655a7b..f940375 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortIntermediateFileMerger.java
@@ -71,7 +71,9 @@ public class SortIntermediateFileMerger {
fileList = procFiles.toArray(new File[procFiles.size()]);
this.procFiles = new ArrayList<File>();
}
- LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+ }
startIntermediateMerging(fileList);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 9c7a5ff..0fc1d64 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -538,8 +538,10 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
DataOutputStream dataOutputStream = null;
DataInputStream dataInputStream = null;
try {
- LOGGER.debug(
- "HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize + " (bytes");
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize
+ + " (bytes");
+ }
dataOutputStream = FileFactory
.getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
bufferSize, blockSize);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4a79a86a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 740b5dc..2614b8f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -475,4 +475,16 @@ public final class CarbonDataProcessorUtil {
return batchSortSizeInMb;
}
+ /**
+ * the method prepares and return the message mentioning the reason of badrecord
+ *
+ * @param columnName
+ * @param dataType
+ * @return
+ */
+ public static String prepareFailureReason(String columnName, DataType dataType) {
+ return "The value with column name " + columnName + " and column data type " + dataType
+ .getName() + " is not a valid " + dataType + " type.";
+ }
+
}
\ No newline at end of file
[2/2] carbondata git commit: [CARBONDATA-1049] avoid logging raw data
into driver and executor log. This closes #909
Posted by ra...@apache.org.
[CARBONDATA-1049] avoid logging raw data into driver and executor log. This closes #909
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f53ab4b2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f53ab4b2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f53ab4b2
Branch: refs/heads/master
Commit: f53ab4b2b09cce919ac3eab82cc58127909521cf
Parents: 657902d 4a79a86
Author: ravipesala <ra...@gmail.com>
Authored: Fri Jun 9 18:59:46 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Jun 9 18:59:46 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/cache/CarbonLRUCache.java | 10 +++++---
.../DateDirectDictionaryGenerator.java | 13 ++++++----
.../TimeStampDirectDictionaryGenerator.java | 15 +++++++-----
.../core/mutate/DeleteDeltaBlockDetails.java | 4 +++-
.../core/mutate/SegmentUpdateDetails.java | 6 +++--
.../data/BlockletDeleteDeltaCacheLoader.java | 8 +++++--
.../scan/filter/FilterExpressionProcessor.java | 19 ++++++++-------
.../carbondata/core/scan/filter/FilterUtil.java | 4 +++-
.../carbondata/core/util/DataTypeUtil.java | 8 +++----
.../badrecordloger/BadRecordLoggerTest.scala | 2 +-
.../newflow/converter/BadRecordLogHolder.java | 21 ++++++++++++++++
.../impl/DictionaryFieldConverterImpl.java | 12 ++++++----
.../DirectDictionaryFieldConverterImpl.java | 18 +++++++-------
.../impl/MeasureFieldConverterImpl.java | 25 +++++++++++++-------
.../impl/NonDictionaryFieldConverterImpl.java | 12 ++++++----
.../converter/impl/RowConverterImpl.java | 8 +++++--
.../unsafe/merger/UnsafeIntermediateMerger.java | 10 +++++---
.../sortandgroupby/sortdata/SortDataRows.java | 8 +++++--
.../sortdata/SortIntermediateFileMerger.java | 4 +++-
.../store/writer/AbstractFactDataWriter.java | 6 +++--
.../util/CarbonDataProcessorUtil.java | 12 ++++++++++
21 files changed, 156 insertions(+), 69 deletions(-)
----------------------------------------------------------------------