You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/10/24 13:59:27 UTC
[1/2] incubator-carbondata git commit: Correct Some Spelling mistakes
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 0e82b6501 -> 2fed52639
Correct Some Spelling mistakes
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fb8297ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fb8297ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fb8297ca
Branch: refs/heads/master
Commit: fb8297ca94145fdf042bb6e127c131faeb747fc2
Parents: 0e82b65
Author: lion-x <xl...@gmail.com>
Authored: Fri Oct 14 11:26:33 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Oct 24 21:34:48 2016 +0800
----------------------------------------------------------------------
.../hadoop/test/util/StoreCreator.java | 36 ++++++-------
.../carbondata/spark/load/CarbonLoaderUtil.java | 53 +++++++++++---------
.../carbondata/test/util/StoreCreator.java | 36 ++++++-------
3 files changed, 65 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fb8297ca/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 7c9e170..27a1a3e 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -347,16 +347,16 @@ public class StoreCreator {
path.delete();
}
- DataProcessTaskStatus schmaModel = new DataProcessTaskStatus(databaseName, tableName);
- schmaModel.setCsvFilePath(loadModel.getFactFilePath());
+ DataProcessTaskStatus dataProcessTaskStatus = new DataProcessTaskStatus(databaseName, tableName);
+ dataProcessTaskStatus.setCsvFilePath(loadModel.getFactFilePath());
SchemaInfo info = new SchemaInfo();
BlockDetails blockDetails = new BlockDetails(loadModel.getFactFilePath(),
0, new File(loadModel.getFactFilePath()).length(), new String[] {"localhost"});
GraphGenerator.blockInfo.put("qwqwq", new BlockDetails[] { blockDetails });
- schmaModel.setBlocksID("qwqwq");
- schmaModel.setEscapeCharacter("\\");
- schmaModel.setQuoteCharacter("\"");
- schmaModel.setCommentCharacter("#");
+ dataProcessTaskStatus.setBlocksID("qwqwq");
+ dataProcessTaskStatus.setEscapeCharacter("\\");
+ dataProcessTaskStatus.setQuoteCharacter("\"");
+ dataProcessTaskStatus.setCommentCharacter("#");
info.setDatabaseName(databaseName);
info.setTableName(tableName);
info.setSerializationNullFormat(
@@ -366,10 +366,10 @@ public class StoreCreator {
info.setBadRecordsLoggerAction(
TableOptionConstant.BAD_RECORDS_ACTION.getName() + "," + "force");
- generateGraph(schmaModel, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
+ generateGraph(dataProcessTaskStatus, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
loadModel.getLoadMetadataDetails());
- DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
+ DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
.executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
info, "0", loadModel.getSchema());
@@ -449,7 +449,7 @@ public class StoreCreator {
/**
* generate graph
*
- * @param schmaModel
+ * @param dataProcessTaskStatus
* @param info
* @param tableName
* @param partitionID
@@ -458,20 +458,20 @@ public class StoreCreator {
* @param loadMetadataDetails
* @throws GraphGeneratorException
*/
- private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
+ private static void generateGraph(IDataProcessStatus dataProcessTaskStatus, SchemaInfo info,
String tableName, String partitionID, CarbonDataLoadSchema schema, String factStoreLocation,
List<LoadMetadataDetails> loadMetadataDetails)
throws GraphGeneratorException {
DataLoadModel model = new DataLoadModel();
- model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
+ model.setCsvLoad(null != dataProcessTaskStatus.getCsvFilePath() || null != dataProcessTaskStatus.getFilesToProcess());
model.setSchemaInfo(info);
- model.setTableName(schmaModel.getTableName());
+ model.setTableName(dataProcessTaskStatus.getTableName());
model.setTaskNo("1");
- model.setBlocksID(schmaModel.getBlocksID());
+ model.setBlocksID(dataProcessTaskStatus.getBlocksID());
model.setFactTimeStamp(readCurrentTime());
- model.setEscapeCharacter(schmaModel.getEscapeCharacter());
- model.setQuoteCharacter(schmaModel.getQuoteCharacter());
- model.setCommentCharacter(schmaModel.getCommentCharacter());
+ model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
+ model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
+ model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
model.setLoadNames(
CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails));
@@ -479,8 +479,8 @@ public class StoreCreator {
.getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails));
}
boolean hdfsReadMode =
- schmaModel.getCsvFilePath() != null && schmaModel.getCsvFilePath().startsWith("hdfs:");
- int allocate = null != schmaModel.getCsvFilePath() ? 1 : schmaModel.getFilesToProcess().size();
+ dataProcessTaskStatus.getCsvFilePath() != null && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:");
+ int allocate = null != dataProcessTaskStatus.getCsvFilePath() ? 1 : dataProcessTaskStatus.getFilesToProcess().size();
String outputLocation = CarbonProperties.getInstance()
.getProperty("store_output_location", "../carbon-store/system/carbon/etl");
GraphGenerator generator =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fb8297ca/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 8bc5fdc..7c345e4 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -113,13 +113,14 @@ public final class CarbonLoaderUtil {
}
- private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
+ private static void generateGraph(IDataProcessStatus dataProcessTaskStatus, SchemaInfo info,
CarbonLoadModel loadModel, String outputLocation)
throws GraphGeneratorException {
DataLoadModel model = new DataLoadModel();
- model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
+ model.setCsvLoad(null != dataProcessTaskStatus.getCsvFilePath()
+ || null != dataProcessTaskStatus.getFilesToProcess());
model.setSchemaInfo(info);
- model.setTableName(schmaModel.getTableName());
+ model.setTableName(dataProcessTaskStatus.getTableName());
List<LoadMetadataDetails> loadMetadataDetails = loadModel.getLoadMetadataDetails();
if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
model.setLoadNames(
@@ -127,17 +128,20 @@ public final class CarbonLoaderUtil {
model.setModificationOrDeletionTime(CarbonDataProcessorUtil
.getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails));
}
- model.setBlocksID(schmaModel.getBlocksID());
- model.setEscapeCharacter(schmaModel.getEscapeCharacter());
- model.setQuoteCharacter(schmaModel.getQuoteCharacter());
- model.setCommentCharacter(schmaModel.getCommentCharacter());
- model.setRddIteratorKey(schmaModel.getRddIteratorKey());
+ model.setBlocksID(dataProcessTaskStatus.getBlocksID());
+ model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
+ model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
+ model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
+ model.setRddIteratorKey(dataProcessTaskStatus.getRddIteratorKey());
model.setTaskNo(loadModel.getTaskNo());
model.setFactTimeStamp(loadModel.getFactTimeStamp());
model.setMaxColumns(loadModel.getMaxColumns());
boolean hdfsReadMode =
- schmaModel.getCsvFilePath() != null && schmaModel.getCsvFilePath().startsWith("hdfs:");
- int allocate = null != schmaModel.getCsvFilePath() ? 1 : schmaModel.getFilesToProcess().size();
+ dataProcessTaskStatus.getCsvFilePath() != null
+ && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:");
+ int allocate =
+ null != dataProcessTaskStatus.getCsvFilePath()
+ ? 1 : dataProcessTaskStatus.getFilesToProcess().size();
GraphGenerator generator = new GraphGenerator(model, hdfsReadMode, loadModel.getPartitionId(),
loadModel.getStorePath(), allocate,
loadModel.getCarbonDataLoadSchema(), loadModel.getSegmentId(), outputLocation);
@@ -174,21 +178,22 @@ public final class CarbonLoaderUtil {
path.delete();
}
- DataProcessTaskStatus schmaModel = new DataProcessTaskStatus(databaseName, tableName);
- schmaModel.setCsvFilePath(loadModel.getFactFilePath());
- schmaModel.setDimCSVDirLoc(loadModel.getDimFolderPath());
+ DataProcessTaskStatus dataProcessTaskStatus
+ = new DataProcessTaskStatus(databaseName, tableName);
+ dataProcessTaskStatus.setCsvFilePath(loadModel.getFactFilePath());
+ dataProcessTaskStatus.setDimCSVDirLoc(loadModel.getDimFolderPath());
if (loadModel.isDirectLoad()) {
- schmaModel.setFilesToProcess(loadModel.getFactFilesToProcess());
- schmaModel.setDirectLoad(true);
- schmaModel.setCsvDelimiter(loadModel.getCsvDelimiter());
- schmaModel.setCsvHeader(loadModel.getCsvHeader());
+ dataProcessTaskStatus.setFilesToProcess(loadModel.getFactFilesToProcess());
+ dataProcessTaskStatus.setDirectLoad(true);
+ dataProcessTaskStatus.setCsvDelimiter(loadModel.getCsvDelimiter());
+ dataProcessTaskStatus.setCsvHeader(loadModel.getCsvHeader());
}
- schmaModel.setBlocksID(loadModel.getBlocksID());
- schmaModel.setEscapeCharacter(loadModel.getEscapeChar());
- schmaModel.setQuoteCharacter(loadModel.getQuoteChar());
- schmaModel.setCommentCharacter(loadModel.getCommentChar());
- schmaModel.setRddIteratorKey(loadModel.getRddIteratorKey());
+ dataProcessTaskStatus.setBlocksID(loadModel.getBlocksID());
+ dataProcessTaskStatus.setEscapeCharacter(loadModel.getEscapeChar());
+ dataProcessTaskStatus.setQuoteCharacter(loadModel.getQuoteChar());
+ dataProcessTaskStatus.setCommentCharacter(loadModel.getCommentChar());
+ dataProcessTaskStatus.setRddIteratorKey(loadModel.getRddIteratorKey());
SchemaInfo info = new SchemaInfo();
info.setDatabaseName(databaseName);
@@ -200,9 +205,9 @@ public final class CarbonLoaderUtil {
info.setBadRecordsLoggerEnable(loadModel.getBadRecordsLoggerEnable());
info.setBadRecordsLoggerAction(loadModel.getBadRecordsAction());
- generateGraph(schmaModel, info, loadModel, outPutLoc);
+ generateGraph(dataProcessTaskStatus, info, loadModel, outPutLoc);
- DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
+ DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
.executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
info, loadModel.getPartitionId(), loadModel.getCarbonDataLoadSchema());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fb8297ca/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 2faabaf..22c2917 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -357,23 +357,23 @@ public class StoreCreator {
path.delete();
}
- DataProcessTaskStatus schmaModel = new DataProcessTaskStatus(databaseName, tableName);
- schmaModel.setCsvFilePath(loadModel.getFactFilePath());
+ DataProcessTaskStatus dataProcessTaskStatus = new DataProcessTaskStatus(databaseName, tableName);
+ dataProcessTaskStatus.setCsvFilePath(loadModel.getFactFilePath());
SchemaInfo info = new SchemaInfo();
BlockDetails blockDetails = new BlockDetails(loadModel.getFactFilePath(),
0, new File(loadModel.getFactFilePath()).length(), new String[] {"localhost"});
GraphGenerator.blockInfo.put("qwqwq", new BlockDetails[] { blockDetails });
- schmaModel.setBlocksID("qwqwq");
- schmaModel.setEscapeCharacter("\\");
- schmaModel.setQuoteCharacter("\"");
- schmaModel.setCommentCharacter("#");
+ dataProcessTaskStatus.setBlocksID("qwqwq");
+ dataProcessTaskStatus.setEscapeCharacter("\\");
+ dataProcessTaskStatus.setQuoteCharacter("\"");
+ dataProcessTaskStatus.setCommentCharacter("#");
info.setDatabaseName(databaseName);
info.setTableName(tableName);
- generateGraph(schmaModel, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
+ generateGraph(dataProcessTaskStatus, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
loadModel.getLoadMetadataDetails());
- DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
+ DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
graphExecuter
.executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
info, "0", loadModel.getSchema());
@@ -453,7 +453,7 @@ public class StoreCreator {
/**
* generate graph
*
- * @param schmaModel
+ * @param dataProcessTaskStatus
* @param info
* @param tableName
* @param partitionID
@@ -462,20 +462,20 @@ public class StoreCreator {
* @param loadMetadataDetails
* @throws GraphGeneratorException
*/
- private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
+ private static void generateGraph(IDataProcessStatus dataProcessTaskStatus, SchemaInfo info,
String tableName, String partitionID, CarbonDataLoadSchema schema, String factStoreLocation,
List<LoadMetadataDetails> loadMetadataDetails)
throws GraphGeneratorException {
DataLoadModel model = new DataLoadModel();
- model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
+ model.setCsvLoad(null != dataProcessTaskStatus.getCsvFilePath() || null != dataProcessTaskStatus.getFilesToProcess());
model.setSchemaInfo(info);
- model.setTableName(schmaModel.getTableName());
+ model.setTableName(dataProcessTaskStatus.getTableName());
model.setTaskNo("1");
- model.setBlocksID(schmaModel.getBlocksID());
+ model.setBlocksID(dataProcessTaskStatus.getBlocksID());
model.setFactTimeStamp(readCurrentTime());
- model.setEscapeCharacter(schmaModel.getEscapeCharacter());
- model.setQuoteCharacter(schmaModel.getQuoteCharacter());
- model.setCommentCharacter(schmaModel.getCommentCharacter());
+ model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
+ model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
+ model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
model.setLoadNames(
CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails));
@@ -483,8 +483,8 @@ public class StoreCreator {
.getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails));
}
boolean hdfsReadMode =
- schmaModel.getCsvFilePath() != null && schmaModel.getCsvFilePath().startsWith("hdfs:");
- int allocate = null != schmaModel.getCsvFilePath() ? 1 : schmaModel.getFilesToProcess().size();
+ dataProcessTaskStatus.getCsvFilePath() != null && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:");
+ int allocate = null != dataProcessTaskStatus.getCsvFilePath() ? 1 : dataProcessTaskStatus.getFilesToProcess().size();
String outputLocation = CarbonProperties.getInstance()
.getProperty("store_output_location", "../carbon-store/system/carbon/etl");
GraphGenerator generator =
[2/2] incubator-carbondata git commit: [CARBONDATA-334] Correct Some
Spelling Mistakes This closes #238
Posted by ja...@apache.org.
[CARBONDATA-334] Correct Some Spelling Mistakes This closes #238
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2fed5263
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2fed5263
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2fed5263
Branch: refs/heads/master
Commit: 2fed526391bf9a99aa12ec6981101cbebbddb0ca
Parents: 0e82b65 fb8297c
Author: jackylk <ja...@huawei.com>
Authored: Mon Oct 24 21:59:09 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Oct 24 21:59:09 2016 +0800
----------------------------------------------------------------------
.../hadoop/test/util/StoreCreator.java | 36 ++++++-------
.../carbondata/spark/load/CarbonLoaderUtil.java | 53 +++++++++++---------
.../carbondata/test/util/StoreCreator.java | 36 ++++++-------
3 files changed, 65 insertions(+), 60 deletions(-)
----------------------------------------------------------------------