You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by aj...@apache.org on 2020/09/15 12:07:40 UTC
[carbondata] branch master updated: [CARBONDATA-3928] Removed
records from exception message.
This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 7caa530 [CARBONDATA-3928] Removed records from exception message.
7caa530 is described below
commit 7caa530edde9709f3e7fae0d2f2e2956cf7e77cb
Author: Nihal ojha <ni...@gmail.com>
AuthorDate: Fri Sep 11 13:01:37 2020 +0530
[CARBONDATA-3928] Removed records from exception message.
Why is this PR needed?
Currently, when the string length exceeds 32000 and bad record action as default then
records are thrown in exception message which violates security concern.
What changes were proposed in this PR?
Removed the records from exception message and only printed in logger file and redirected csv file.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #3921
---
.../carbondata/core/constants/CarbonCommonConstants.java | 2 +-
.../org/apache/carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../spark/testsuite/dataload/TestLoadDataGeneral.scala | 4 ++--
.../carbondata/processing/datatypes/PrimitiveDataType.java | 6 +++---
.../processing/loading/converter/impl/RowConverterImpl.java | 10 ++--------
5 files changed, 9 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index d34f357..e497c73 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -2535,7 +2535,7 @@ public final class CarbonCommonConstants {
public static final String CARBON_INDEXSERVER_TEMPFOLDER_DELETETIME_DEFAULT = "10800000";
public static final String STRING_LENGTH_EXCEEDED_MESSAGE =
- "Record %s of column %s exceeded " + MAX_CHARS_PER_COLUMN_DEFAULT +
+ "Record of column %s exceeded " + MAX_CHARS_PER_COLUMN_DEFAULT +
" characters. Please consider long string data type.";
/**
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 2806376..3e1374e 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -76,7 +76,7 @@ object CarbonScalaUtil {
} catch {
case e: Exception =>
if (e.getMessage.startsWith(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE)) {
- val msg = CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE.format(row,
+ val msg = CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE.format(
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getCreateOrderColumn
.get(idx).getColName)
LOGGER.error(msg, e)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index aaaa88d..a5a2c41 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -236,7 +236,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
exception = intercept[Exception] {
sql(s"insert into longerthan32kchar values('33000', '$longChar', 4)")
}
- assert(exception.getMessage.contains(s"Record [33000, $longChar, 4] of column dim2 exceeded " +
+ assert(exception.getMessage.contains(s"Record of column dim2 exceeded " +
s"${CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT} characters. Please consider long string data type."))
// Update strings of length greater than 32000
sql(s"insert into longerthan32kchar values('ok', 'hi', 1)")
@@ -244,7 +244,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach {
sql(s"update longerthan32kchar set(longerthan32kchar.dim2)=('$longChar') " +
"where longerthan32kchar.mes1=1").show()
}
- assert(exception.getMessage.contains(s"Record [ok, $longChar, 1] of column dim2 exceeded " +
+ assert(exception.getMessage.contains(s"Record of column dim2 exceeded " +
s"${CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT} characters. Please consider long string data type."))
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "false")
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index d9f3e8c..1a09a5d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -302,7 +302,7 @@ public class PrimitiveDataType implements GenericDataType<Object> {
if (this.carbonDimension.getDataType() == DataTypes.STRING
&& value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE,
- input.toString(), this.carbonDimension.getColName()));
+ this.carbonDimension.getColName()));
updateNullValue(dataOutputStream, logHolder);
return;
}
@@ -339,7 +339,7 @@ public class PrimitiveDataType implements GenericDataType<Object> {
if (this.carbonDimension.getDataType() == DataTypes.STRING && input instanceof String
&& ((String)input).length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE,
- input.toString(), this.carbonDimension.getColName()));
+ this.carbonDimension.getColName()));
updateNullValue(dataOutputStream, logHolder);
return;
}
@@ -348,7 +348,7 @@ public class PrimitiveDataType implements GenericDataType<Object> {
if (this.carbonDimension.getDataType() == DataTypes.STRING
&& value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE,
- input.toString(), this.carbonDimension.getColName()));
+ this.carbonDimension.getColName()));
updateNullValue(dataOutputStream, logHolder);
return;
}
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
index 5d2226a..28bfa91 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.processing.loading.converter.impl;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -119,10 +118,6 @@ public class RowConverterImpl implements RowConverter {
.getTableProperties();
String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX);
boolean isSpatialColumn = false;
- Object[] rawData = row.getRawData();
- if (rawData == null) {
- rawData = row.getData() == null ? null : row.getData().clone();
- }
for (int i = 0; i < fieldConverters.length; i++) {
if (spatialProperty != null) {
isSpatialColumn = fieldConverters[i].getDataField().getColumn().getColName()
@@ -137,10 +132,9 @@ public class RowConverterImpl implements RowConverter {
if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) {
String reason = logHolder.getReason();
if (reason.equalsIgnoreCase(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE)) {
- reason = String.format(reason, Arrays.toString(rawData),
- this.fields[i].getColumn().getColName());
+ reason = String.format(reason, this.fields[i].getColumn().getColName());
}
- badRecordLogger.addBadRecordsToBuilder(rawData, reason);
+ badRecordLogger.addBadRecordsToBuilder(row.getRawData(), reason);
if (badRecordLogger.isDataLoadFail()) {
String error = "Data load failed due to bad record: " + reason;
if (!badRecordLogger.isBadRecordLoggerEnable()) {