You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/05/03 08:53:18 UTC
carbondata git commit: [CARBONDATA-2410] Error message correction
when column value length exceeds 320000 character
Repository: carbondata
Updated Branches:
refs/heads/master 167b6c004 -> 2c095542b
[CARBONDATA-2410] Error message correction when column value length exceeds 320000 character
The raw data is displayed in the client, to avoid the same the exception message is intercepted and sent back to driver
This closes #2243
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2c095542
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2c095542
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2c095542
Branch: refs/heads/master
Commit: 2c095542bd3969c28fe13866ec5bad6a94cf6b3d
Parents: 167b6c0
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Fri Apr 27 19:08:03 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Thu May 3 14:26:29 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/constants/CarbonCommonConstants.java | 7 +++++++
.../apache/carbondata/spark/util/GlobalDictionaryUtil.scala | 4 +++-
.../processing/loading/csvinput/CSVInputFormat.java | 3 +++
.../carbondata/processing/util/CarbonDataProcessorUtil.java | 4 +++-
4 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b902359..553698a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1477,6 +1477,13 @@ public final class CarbonCommonConstants {
@CarbonProperty public static final String CARBON_SKIP_EMPTY_LINE = "carbon.skip.empty.line";
public static final String CARBON_SKIP_EMPTY_LINE_DEFAULT = "false";
+ /**
+ * Configures the parser/writer to limit the length of displayed contents being parsed/written
+ * in the exception message when an error occurs.
+ * Here {@code 0} means no exceptions will include the content being manipulated in their
+ * attributes.
+ */
+ public static final int CARBON_ERROR_CONTENT_LENGTH = 0;
/**
* if the byte size of streaming segment reach this value,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index fb9ecac..5129e59 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -824,9 +824,11 @@ object GlobalDictionaryUtil {
def trimErrorMessage(input: String): String = {
var errorMessage: String = null
if (input != null && input.contains("TextParsingException:")) {
- if (input.split("Hint").length > 0 &&
+ if (input.split("Hint").length > 1 &&
input.split("Hint")(0).split("TextParsingException: ").length > 1) {
errorMessage = input.split("Hint")(0).split("TextParsingException: ")(1)
+ } else if (input.split("Parser Configuration:").length > 1) {
+ errorMessage = input.split("Parser Configuration:")(0)
}
} else if (input != null && input.contains("Exception:")) {
errorMessage = input.split("Exception: ")(1).split("\n")(0)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
index aebaf3b..2e3479c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
@@ -210,6 +210,9 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri
parserSettings.setMaxColumns(Integer.parseInt(maxColumns));
parserSettings.getFormat().setQuote(job.get(QUOTE, QUOTE_DEFAULT).charAt(0));
parserSettings.getFormat().setQuoteEscape(job.get(ESCAPE, ESCAPE_DEFAULT).charAt(0));
+ // setting the content length to to limit the length of displayed contents being parsed/written
+ // in the exception message when an error occurs.
+ parserSettings.setErrorContentLength(CarbonCommonConstants.CARBON_ERROR_CONTENT_LENGTH);
return parserSettings;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 6ba05a2..f47853e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -592,8 +592,10 @@ public final class CarbonDataProcessorUtil {
public static String trimErrorMessage(String input) {
String errorMessage = input;
if (input != null) {
- if (input.split("Hint").length > 0) {
+ if (input.split("Hint").length > 1) {
errorMessage = input.split("Hint")[0];
+ } else if (input.split("Parser Configuration:").length > 1) {
+ errorMessage = input.split("Parser Configuration:")[0];
}
}
return errorMessage;