You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/08/30 07:43:07 UTC
[1/2] incubator-carbondata git commit: Handle all dictionary
exception more properly
Repository: incubator-carbondata
Updated Branches:
refs/heads/master b5b0bad14 -> bbc4ffabd
Handle all dictionary exception more properly
modifed by suggestions
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/6187b13f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/6187b13f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/6187b13f
Branch: refs/heads/master
Commit: 6187b13fd6cb304a5900c2c6f21c26d07c59b9fa
Parents: b5b0bad
Author: foryou2030 <fo...@126.com>
Authored: Sat Aug 27 14:42:16 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Aug 30 13:07:26 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +-
.../store/filesystem/LocalCarbonFile.java | 5 +-
.../spark/util/GlobalDictionaryUtil.scala | 54 ++++++++++++--------
3 files changed, 39 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6187b13f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 69d9c3c..c4b412e 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -199,9 +199,9 @@ public final class CarbonCommonConstants {
*/
public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
/**
- * BLANK_LINE_FLAG
+ * DEFAULT_COLUMN_NAME
*/
- public static final String BLANK_LINE_FLAG = "@NU#LL$!BLANKLINE";
+ public static final String DEFAULT_COLUMN_NAME = "@NU#LL$!COLUMN";
/**
* FILE STATUS IN-PROGRESS
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6187b13f/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
index f46aeed..406f6d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
@@ -86,7 +86,10 @@ public class LocalCarbonFile implements CarbonFile {
}
@Override public boolean exists() {
- return file.exists();
+ if (file != null) {
+ return file.exists();
+ }
+ return false;
}
@Override public String getCanonicalPath() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/6187b13f/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index c894594..ebed94b 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.util
-import java.io.IOException
+import java.io.{FileNotFoundException, IOException}
import java.nio.charset.Charset
import java.util.regex.Pattern
@@ -592,29 +592,31 @@ object GlobalDictionaryUtil extends Logging {
val basicRdd = sqlContext.sparkContext.textFile(allDictionaryPath)
.map(x => {
val tokens = x.split("" + CSVWriter.DEFAULT_SEPARATOR)
- var index: Int = 0
+ if (tokens.size != 2) {
+ logError("Read a bad dictionary record: " + x)
+ }
+ var columnName: String = CarbonCommonConstants.DEFAULT_COLUMN_NAME
var value: String = ""
try {
- index = tokens(0).toInt
+ columnName = csvFileColumns(tokens(0).toInt)
value = tokens(1)
} catch {
case ex: Exception =>
- logError("read a bad dictionary record" + x)
+ logError("Reset bad dictionary record as default value")
}
- (index, value)
+ (columnName, value)
})
+
// group by column index, and filter required columns
val requireColumnsList = requireColumns.toList
allDictionaryRdd = basicRdd
.groupByKey()
- .map(x => (csvFileColumns(x._1), x._2))
.filter(x => requireColumnsList.contains(x._1))
} catch {
case ex: Exception =>
- logError("read local dictionary files failed")
+ logError("Read dictionary files failed. Caused by: " + ex.getMessage)
throw ex
}
-
allDictionaryRdd
}
@@ -630,22 +632,32 @@ object GlobalDictionaryUtil extends Logging {
// filepath regex, look like "/path/*.dictionary"
if (filePath.getName.startsWith("*")) {
val dictExt = filePath.getName.substring(1)
- val listFiles = filePath.getParentFile.listFiles()
- if (listFiles.exists(file =>
- file.getName.endsWith(dictExt) && file.getSize > 0)) {
- true
+ if (filePath.getParentFile.exists()) {
+ val listFiles = filePath.getParentFile.listFiles()
+ if (listFiles.exists(file =>
+ file.getName.endsWith(dictExt) && file.getSize > 0)) {
+ true
+ } else {
+ logWarning("No dictionary files found or empty dictionary files! " +
+ "Won't generate new dictionary.")
+ false
+ }
} else {
- logInfo("No dictionary files found or empty dictionary files! " +
- "Won't generate new dictionary.")
- false
+ throw new FileNotFoundException(
+ "The given dictionary file path is not found!")
}
} else {
- if (filePath.exists() && filePath.getSize > 0) {
- true
+ if (filePath.exists()) {
+ if (filePath.getSize > 0) {
+ true
+ } else {
+ logWarning("No dictionary files found or empty dictionary files! " +
+ "Won't generate new dictionary.")
+ false
+ }
} else {
- logInfo("No dictionary files found or empty dictionary files! " +
- "Won't generate new dictionary.")
- false
+ throw new FileNotFoundException(
+ "The given dictionary file path is not found!")
}
}
}
@@ -733,7 +745,7 @@ object GlobalDictionaryUtil extends Logging {
}
}
} else {
- logInfo("Generate global dictionary from all dictionary files!")
+ logInfo("Generate global dictionary from dictionary files!")
val isNonempty = validateAllDictionaryPath(allDictionaryPath)
if(isNonempty) {
// fill the map[columnIndex -> columnName]
[2/2] incubator-carbondata git commit: Handle all dictionary
exception more properly This closes #100
Posted by ra...@apache.org.
Handle all dictionary exception more properly This closes #100
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/bbc4ffab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/bbc4ffab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/bbc4ffab
Branch: refs/heads/master
Commit: bbc4ffabd7acf3fe684665128fb9e19a524c5436
Parents: b5b0bad 6187b13
Author: ravipesala <ra...@gmail.com>
Authored: Tue Aug 30 13:12:25 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Aug 30 13:12:25 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +-
.../store/filesystem/LocalCarbonFile.java | 5 +-
.../spark/util/GlobalDictionaryUtil.scala | 54 ++++++++++++--------
3 files changed, 39 insertions(+), 24 deletions(-)
----------------------------------------------------------------------