You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/30 14:23:41 UTC
[1/2] incubator-carbondata git commit: give a proper error message
for wrong delimiter or fileheader
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 78c0fb7aa -> 2f13668c2
give a proper error message for wrong delimiter or fileheader
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/929acf6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/929acf6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/929acf6b
Branch: refs/heads/master
Commit: 929acf6ba575b6293c511a6d505e73d2983bc514
Parents: 78c0fb7
Author: QiangCai <da...@gmail.com>
Authored: Fri Aug 26 10:57:53 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Aug 30 19:51:51 2016 +0530
----------------------------------------------------------------------
.../carbondata/spark/util/GlobalDictionaryUtil.scala | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/929acf6b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index acd69bd..4a2cd84 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -727,6 +727,12 @@ object GlobalDictionaryUtil extends Logging {
generatePredefinedColDictionary(colDictFilePath, table,
dimensions, carbonLoadModel, sqlContext, hdfsLocation, dictfolderPath)
}
+ if (headers.length > df.columns.length) {
+ val msg = "The number of columns in the file header do not match the number of " +
+ "columns in the data file; Either delimiter or fileheader provided is not correct"
+ logError(msg)
+ throw new DataLoadingException(msg)
+ }
// use fact file to generate global dict
val (requireDimension, requireColumnNames) = pruneDimensions(dimensions,
headers, df.columns)
@@ -743,7 +749,7 @@ object GlobalDictionaryUtil extends Logging {
// check result status
checkStatus(carbonLoadModel, sqlContext, model, statusList)
} else {
- logInfo("have no column need to generate global dictionary in Fact file")
+ logInfo("No column found for generating global dictionary in source data files")
}
// generate global dict from dimension file
if (carbonLoadModel.getDimFolderPath != null) {
@@ -802,7 +808,7 @@ object GlobalDictionaryUtil extends Logging {
}
} catch {
case ex: Exception =>
- logError("generate global dictionary failed")
+ logError("generate global dictionary failed", ex)
throw ex
}
}
[2/2] incubator-carbondata git commit: [CARBONDATA-180]give proper
error message when dataloading with wrong delimiter or fileheader This closes
#97
Posted by ra...@apache.org.
[CARBONDATA-180]give proper error message when dataloading with wrong delimiter or fileheader This closes #97
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2f13668c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2f13668c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2f13668c
Branch: refs/heads/master
Commit: 2f13668c2154d65c9a809c1a292f695c857e8750
Parents: 78c0fb7 929acf6
Author: ravipesala <ra...@gmail.com>
Authored: Tue Aug 30 19:52:59 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Aug 30 19:52:59 2016 +0530
----------------------------------------------------------------------
.../carbondata/spark/util/GlobalDictionaryUtil.scala | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------