You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/12/05 16:29:25 UTC
carbondata git commit: [CARBONDATA-1831] BAD_RECORDS: Data Loading
with Action as Redirect & logger enable is not logging the logs in the
defined path.
Repository: carbondata
Updated Branches:
refs/heads/master 3d8f09bac -> 4e3151087
[CARBONDATA-1831] BAD_RECORDS: Data Loading with Action as Redirect & logger enable is not logging the logs in the defined path.
Current system expects the carbon.badRecords.location to be absolute path with filescheme.
But, it should support absolute path without file scheme(eg.: hdfs://, viewfs://)
This closes #1609
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4e315108
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4e315108
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4e315108
Branch: refs/heads/master
Commit: 4e315108763858781d0ef2c340e3343e9860b14a
Parents: 3d8f09b
Author: dhatchayani <dh...@gmail.com>
Authored: Mon Dec 4 20:01:06 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Dec 6 00:29:10 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/filesystem/LocalCarbonFile.java | 2 ++
.../scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala | 3 ++-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 5b2cc68..30eb7fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -123,6 +123,7 @@ public class LocalCarbonFile implements CarbonFile {
}
public boolean renameTo(String changetoName) {
+ changetoName = getUpdatedFilePath(changetoName);
return file.renameTo(new File(changetoName));
}
@@ -318,6 +319,7 @@ public class LocalCarbonFile implements CarbonFile {
@Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType fileType)
throws IOException {
path = path.replace("\\", "/");
+ path = getUpdatedFilePath(path);
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 77f75f2..74ed6a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -209,7 +209,7 @@ object DataLoadingUtil {
val single_pass = optionsFinal("single_pass")
val bad_records_logger_enable = optionsFinal("bad_records_logger_enable")
val bad_records_action = optionsFinal("bad_records_action")
- val bad_record_path = optionsFinal("bad_record_path")
+ var bad_record_path = optionsFinal("bad_record_path")
val global_sort_partitions = optionsFinal("global_sort_partitions")
val timestampformat = optionsFinal("timestampformat")
val dateFormat = optionsFinal("dateformat")
@@ -224,6 +224,7 @@ object DataLoadingUtil {
if (bad_records_logger_enable.toBoolean ||
LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
+ bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path)
if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
CarbonException.analysisException("Invalid bad records location.")
}