You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/09/22 05:36:34 UTC
[36/50] [abbrv] incubator-carbondata git commit: Handle all
dictionary exception more properly
Handle all dictionary exception more properly
modifed by suggestions
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/eedfe596
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/eedfe596
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/eedfe596
Branch: refs/heads/branch-0.1
Commit: eedfe5969edbcca8ea9ec0bc18651af7320ed831
Parents: b53a4db
Author: foryou2030 <fo...@126.com>
Authored: Sat Aug 27 14:42:16 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Sep 22 10:30:16 2016 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +-
.../store/filesystem/LocalCarbonFile.java | 5 +-
.../spark/util/GlobalDictionaryUtil.scala | 54 ++++++++++++--------
3 files changed, 39 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 1e603b5..6050719 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -199,9 +199,9 @@ public final class CarbonCommonConstants {
*/
public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
/**
- * BLANK_LINE_FLAG
+ * DEFAULT_COLUMN_NAME
*/
- public static final String BLANK_LINE_FLAG = "@NU#LL$!BLANKLINE";
+ public static final String DEFAULT_COLUMN_NAME = "@NU#LL$!COLUMN";
/**
* FILE STATUS IN-PROGRESS
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
index f46aeed..406f6d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
@@ -86,7 +86,10 @@ public class LocalCarbonFile implements CarbonFile {
}
@Override public boolean exists() {
- return file.exists();
+ if (file != null) {
+ return file.exists();
+ }
+ return false;
}
@Override public String getCanonicalPath() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eedfe596/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index cabedfd..bdd8adc 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.util
-import java.io.IOException
+import java.io.{FileNotFoundException, IOException}
import java.nio.charset.Charset
import java.util.regex.Pattern
@@ -602,29 +602,31 @@ object GlobalDictionaryUtil extends Logging {
val basicRdd = sqlContext.sparkContext.textFile(allDictionaryPath)
.map(x => {
val tokens = x.split("" + CSVWriter.DEFAULT_SEPARATOR)
- var index: Int = 0
+ if (tokens.size != 2) {
+ logError("Read a bad dictionary record: " + x)
+ }
+ var columnName: String = CarbonCommonConstants.DEFAULT_COLUMN_NAME
var value: String = ""
try {
- index = tokens(0).toInt
+ columnName = csvFileColumns(tokens(0).toInt)
value = tokens(1)
} catch {
case ex: Exception =>
- logError("read a bad dictionary record" + x)
+ logError("Reset bad dictionary record as default value")
}
- (index, value)
+ (columnName, value)
})
+
// group by column index, and filter required columns
val requireColumnsList = requireColumns.toList
allDictionaryRdd = basicRdd
.groupByKey()
- .map(x => (csvFileColumns(x._1), x._2))
.filter(x => requireColumnsList.contains(x._1))
} catch {
case ex: Exception =>
- logError("read local dictionary files failed")
+ logError("Read dictionary files failed. Caused by: " + ex.getMessage)
throw ex
}
-
allDictionaryRdd
}
@@ -640,22 +642,32 @@ object GlobalDictionaryUtil extends Logging {
// filepath regex, look like "/path/*.dictionary"
if (filePath.getName.startsWith("*")) {
val dictExt = filePath.getName.substring(1)
- val listFiles = filePath.getParentFile.listFiles()
- if (listFiles.exists(file =>
- file.getName.endsWith(dictExt) && file.getSize > 0)) {
- true
+ if (filePath.getParentFile.exists()) {
+ val listFiles = filePath.getParentFile.listFiles()
+ if (listFiles.exists(file =>
+ file.getName.endsWith(dictExt) && file.getSize > 0)) {
+ true
+ } else {
+ logWarning("No dictionary files found or empty dictionary files! " +
+ "Won't generate new dictionary.")
+ false
+ }
} else {
- logInfo("No dictionary files found or empty dictionary files! " +
- "Won't generate new dictionary.")
- false
+ throw new FileNotFoundException(
+ "The given dictionary file path is not found!")
}
} else {
- if (filePath.exists() && filePath.getSize > 0) {
- true
+ if (filePath.exists()) {
+ if (filePath.getSize > 0) {
+ true
+ } else {
+ logWarning("No dictionary files found or empty dictionary files! " +
+ "Won't generate new dictionary.")
+ false
+ }
} else {
- logInfo("No dictionary files found or empty dictionary files! " +
- "Won't generate new dictionary.")
- false
+ throw new FileNotFoundException(
+ "The given dictionary file path is not found!")
}
}
}
@@ -768,7 +780,7 @@ object GlobalDictionaryUtil extends Logging {
}
}
} else {
- logInfo("Generate global dictionary from all dictionary files!")
+ logInfo("Generate global dictionary from dictionary files!")
val isNonempty = validateAllDictionaryPath(allDictionaryPath)
if(isNonempty) {
var headers = if (StringUtils.isEmpty(carbonLoadModel.getCsvHeader)) {