You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/29 07:41:39 UTC
[1/2] incubator-carbondata git commit: remove unused dimension table
dict
Repository: incubator-carbondata
Updated Branches:
refs/heads/master fd9d1026c -> aaf98edca
remove unused dimension table dict
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ff1bde14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ff1bde14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ff1bde14
Branch: refs/heads/master
Commit: ff1bde146f40551409708285e2b69b75837f20ba
Parents: fd9d102
Author: Jay357089 <li...@huawei.com>
Authored: Mon Dec 26 20:43:23 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 29 15:28:34 2016 +0800
----------------------------------------------------------------------
.../carbondata/spark/load/CarbonLoaderUtil.java | 2 -
.../spark/util/GlobalDictionaryUtil.scala | 25 ---------
.../execution/command/carbonTableSchema.scala | 6 --
.../spark/util/AllDictionaryTestCase.scala | 6 +-
.../AutoHighCardinalityIdentifyTestCase.scala | 2 -
...GlobalDictionaryUtilConcurrentTestCase.scala | 4 +-
.../util/GlobalDictionaryUtilTestCase.scala | 8 +--
.../execution/command/carbonTableSchema.scala | 6 --
.../processing/csvload/DataGraphExecuter.java | 59 --------------------
.../dataprocessor/DataProcessTaskStatus.java | 13 -----
.../dataprocessor/IDataProcessStatus.java | 4 --
.../processing/model/CarbonLoadModel.java | 17 ------
12 files changed, 5 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 0d2ab6f..56ddce2 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -179,7 +179,6 @@ public final class CarbonLoaderUtil {
DataProcessTaskStatus dataProcessTaskStatus
= new DataProcessTaskStatus(databaseName, tableName);
dataProcessTaskStatus.setCsvFilePath(loadModel.getFactFilePath());
- dataProcessTaskStatus.setDimCSVDirLoc(loadModel.getDimFolderPath());
if (loadModel.isDirectLoad()) {
dataProcessTaskStatus.setFilesToProcess(loadModel.getFactFilesToProcess());
dataProcessTaskStatus.setDirectLoad(true);
@@ -194,7 +193,6 @@ public final class CarbonLoaderUtil {
dataProcessTaskStatus.setRddIteratorKey(loadModel.getRddIteratorKey());
dataProcessTaskStatus.setDateFormat(loadModel.getDateFormat());
SchemaInfo info = new SchemaInfo();
-
info.setDatabaseName(databaseName);
info.setTableName(tableName);
info.setAutoAggregateRequest(loadModel.isAggLoadRequest());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index e578488..3274e58 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -772,31 +772,6 @@ object GlobalDictionaryUtil {
} else {
LOGGER.info("No column found for generating global dictionary in source data files")
}
- // generate global dict from dimension file
- if (carbonLoadModel.getDimFolderPath != null) {
- val fileMapArray = carbonLoadModel.getDimFolderPath.split(",")
- for (fileMap <- fileMapArray) {
- val dimTableName = fileMap.split(":")(0)
- var dimDataframe = loadDataFrame(sqlContext, carbonLoadModel)
- val (requireDimensionForDim, requireColumnNamesForDim) =
- pruneDimensions(dimensions, dimDataframe.columns, dimDataframe.columns)
- if (requireDimensionForDim.length >= 1) {
- dimDataframe = dimDataframe.select(requireColumnNamesForDim.head,
- requireColumnNamesForDim.tail: _*)
- val modelforDim = createDictionaryLoadModel(carbonLoadModel, carbonTableIdentifier,
- requireDimensionForDim, storePath, dictfolderPath, false)
- val inputRDDforDim = new CarbonBlockDistinctValuesCombineRDD(
- dimDataframe.rdd, modelforDim)
- .partitionBy(new ColumnPartitioner(modelforDim.primDimensions.length))
- val statusListforDim = new CarbonGlobalDictionaryGenerateRDD(
- inputRDDforDim, modelforDim).collect()
- checkStatus(carbonLoadModel, sqlContext, modelforDim, statusListforDim)
- } else {
- LOGGER.info(s"No columns in dimension table $dimTableName " +
- "to generate global dictionary")
- }
- }
- }
} else {
LOGGER.info("Generate global dictionary from dictionary files!")
val isNonempty = validateAllDictionaryPath(allDictionaryPath)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 8ad1203..d13346a 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -304,12 +304,6 @@ case class LoadTable(
carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
carbonLoadModel.setStorePath(relation.tableMeta.storePath)
- if (dimFilesPath.isEmpty) {
- carbonLoadModel.setDimFolderPath(null)
- } else {
- val x = dimFilesPath.map(f => f.table + ":" + CarbonUtil.checkAndAppendHDFSUrl(f.loadPath))
- carbonLoadModel.setDimFolderPath(x.mkString(","))
- }
val table = relation.tableMeta.carbonTable
carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 90cf00a..24e478f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -43,7 +43,6 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
def buildCarbonLoadModel(relation: CarbonRelation,
filePath: String,
- dimensionFilePath: String,
header: String,
allDictFilePath: String): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
@@ -55,7 +54,6 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
carbonLoadModel.setTableName(table.getFactTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setDimFolderPath(dimensionFilePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -115,7 +113,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
test("Support generate global dictionary from all dictionary files") {
val header = "id,name,city,age"
- val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, null, header, sampleAllDictionaryFile)
+ val carbonLoadModel = buildCarbonLoadModel(sampleRelation, null, header, sampleAllDictionaryFile)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
@@ -127,7 +125,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
test("Support generate global dictionary from all dictionary files for complex type") {
val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber"
- val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, null, header, complexAllDictionaryFile)
+ val carbonLoadModel = buildCarbonLoadModel(complexRelation, null, header, complexAllDictionaryFile)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
index e92561e..5c61ab8 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
@@ -46,7 +46,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
def buildCarbonLoadModel(relation: CarbonRelation,
filePath: String,
- dimensionFilePath: String,
header: String): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -58,7 +57,6 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
carbonLoadModel.setTableName(table.getFactTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setDimFolderPath(dimensionFilePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
carbonLoadModel.setComplexDelimiterLevel1("\\$")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index b75b1cc..4210574 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -46,7 +46,6 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
def buildCarbonLoadModel(relation: CarbonRelation,
filePath: String,
- dimensionFilePath: String,
header: String): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -58,7 +57,6 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
carbonLoadModel.setTableName(table.getFactTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setDimFolderPath(dimensionFilePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -113,7 +111,7 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
for (i <- 0 until noOfFiles) {
val filePath: String = workDirectory + s"/src/test/resources/singlecolumn_${10 * (i + 1)}.csv"
files += filePath
- loadModels += buildCarbonLoadModel(sampleRelation, filePath, null, "empid")
+ loadModels += buildCarbonLoadModel(sampleRelation, filePath, "empid")
writedummydata(filePath, 10 * (i + 1))
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
index 89aca03..a7601ee 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
@@ -50,7 +50,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
def buildCarbonLoadModel(relation: CarbonRelation,
filePath: String,
- dimensionFilePath: String,
header: String): CarbonLoadModel = {
val carbonLoadModel = new CarbonLoadModel
carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
@@ -62,7 +61,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
carbonLoadModel.setTableName(table.getFactTableName)
carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setDimFolderPath(dimensionFilePath)
carbonLoadModel.setCsvHeader(header)
carbonLoadModel.setCsvDelimiter(",")
carbonLoadModel.setComplexDelimiterLevel1("\\$")
@@ -156,7 +154,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
test("[issue-80]Global Dictionary Generation") {
- val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null, null)
+ val carbonLoadModel = buildCarbonLoadModel(sampleRelation, filePath, null)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
@@ -174,7 +172,7 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
test("[Issue-190]load csv file without header And support complex type") {
val header = "deviceInformationId,channelsId,ROMSize,purchasedate,mobile,MAC,locationinfo," +
"proddate,gamePointId,contractNumber"
- val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, null, header)
+ val carbonLoadModel = buildCarbonLoadModel(complexRelation, complexfilePath, header)
GlobalDictionaryUtil
.generateGlobalDictionary(CarbonHiveContext,
carbonLoadModel,
@@ -188,7 +186,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
// load 1
var carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
complexfilePath1,
- null,
header
)
GlobalDictionaryUtil
@@ -202,7 +199,6 @@ class GlobalDictionaryUtilTestCase extends QueryTest with BeforeAndAfterAll {
// load 2
carbonLoadModel = buildCarbonLoadModel(incrementalLoadTableRelation,
complexfilePath2,
- null,
header
)
GlobalDictionaryUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index a851d47..cafd081 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -355,12 +355,6 @@ case class LoadTable(
carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
carbonLoadModel.setStorePath(relation.tableMeta.storePath)
- if (dimFilesPath.isEmpty) {
- carbonLoadModel.setDimFolderPath(null)
- } else {
- val x = dimFilesPath.map(f => f.table + ":" + CarbonUtil.checkAndAppendHDFSUrl(f.loadPath))
- carbonLoadModel.setDimFolderPath(x.mkString(","))
- }
val table = relation.tableMeta.carbonTable
carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 09e000f..4765277 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -167,7 +167,6 @@ public class DataGraphExecuter {
model.getCsvFilePath() != null && model.getCsvFilePath().startsWith("hdfs:");
trans.setVariable("modifiedDimNames", model.getDimTables());
trans.setVariable("csvInputFilePath", model.getCsvFilePath());
- trans.setVariable("dimFileLocDir", model.getDimCSVDirLoc());
if (hdfsReadMode) {
trans.addParameterDefinition("vfs.hdfs.dfs.client.read.shortcircuit", "true", "");
trans.addParameterDefinition("vfs.hdfs.dfs.domain.socket.path",
@@ -499,64 +498,6 @@ public class DataGraphExecuter {
}
}
}
-
- // Validate the Dimension CSV Files.
- String dimFilesStr = model.getDimCSVDirLoc();
-
- if (null != dimFilesStr && dimFilesStr.length() > 0) {
- String[] dimMapList = model.getDimCSVDirLoc().split(",");
-
- for (String dimFileMap : dimMapList) {
- String tableName = dimFileMap.split(":")[0];
- String dimCSVFileLoc = dimFileMap.substring(tableName.length() + 1);
-
- try {
- if (dimCSVFileLoc != null) {
- FileType fileType = FileFactory.getFileType(dimCSVFileLoc);
- boolean exists = FileFactory.isFileExist(dimCSVFileLoc, fileType);
-
- if (exists) {
- CarbonFile dimCsvFile = FileFactory.getCarbonFile(dimCSVFileLoc, fileType);
-
- String dimFileName = dimCsvFile.getName();
-
- if (dimFileName.endsWith(CarbonCommonConstants.CSV_FILE_EXTENSION)) {
- String dimTableName = dimFileMap.split(":")[0];
-
- validateDimensionCSV(schemaInfo, model.getTableName(), dimTableName, dimCsvFile,
- partitionId, schema, ",");
- } else {
- LOGGER.error(
- "Dimension table file provided to load Dimension tables is not a CSV file : "
- + dimCSVFileLoc);
- throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
- "Dimension table file provided to load Dimension tables is not a CSV file : "
- + dimCSVFileLoc);
- }
- } else {
- LOGGER.error(
- "Dimension table csv file not present in the path provided to load Dimension "
- + "tables : "
- + dimCSVFileLoc);
- throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
- "Dimension table csv file not present in the path provided to load Dimension "
- + "tables : "
- + dimCSVFileLoc);
- }
- }
- } catch (IOException e) {
- LOGGER.error(
- "Dimension table csv file not present in the path provided to load Dimension tables"
- + " : "
- + dimCSVFileLoc);
-
- throw new DataLoadingException(DataProcessorConstants.CSV_VALIDATION_ERRROR_CODE,
- "Dimension table csv file not present in the path provided to load Dimension tables"
- + " : "
- + dimCSVFileLoc);
- }
- }
- }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
index ae0853c..2900353 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/DataProcessTaskStatus.java
@@ -67,11 +67,6 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable {
private String csvFilePath;
- /**
- * dimCSVDirLoc
- */
- private String dimCSVDirLoc;
-
private String dimTables;
private boolean isDirectLoad;
@@ -268,14 +263,6 @@ public class DataProcessTaskStatus implements IDataProcessStatus, Serializable {
return copy;
}
- public String getDimCSVDirLoc() {
- return dimCSVDirLoc;
- }
-
- public void setDimCSVDirLoc(String dimCSVDirLoc) {
- this.dimCSVDirLoc = dimCSVDirLoc;
- }
-
public String getBlocksID() {
return blocksID;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
index a21eb66..d071180 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/IDataProcessStatus.java
@@ -141,10 +141,6 @@ public interface IDataProcessStatus {
*/
void setCsvFilePath(String csvFilePath);
- String getDimCSVDirLoc();
-
- void setDimCSVDirLoc(String dimCSVDirLoc);
-
/**
* @return the dimTables
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff1bde14/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index 10f5197..692e5d7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -307,21 +307,6 @@ public class CarbonLoadModel implements Serializable {
this.colDictFilePath = colDictFilePath;
}
- /**
- * @return the dimFolderPath
- */
- public String getDimFolderPath() {
- return dimFolderPath;
- }
-
- //TODO SIMIAN
-
- /**
- * @param dimFolderPath the dimFolderPath to set
- */
- public void setDimFolderPath(String dimFolderPath) {
- this.dimFolderPath = dimFolderPath;
- }
/**
* get copy with parition
@@ -332,7 +317,6 @@ public class CarbonLoadModel implements Serializable {
public CarbonLoadModel getCopyWithPartition(String uniqueId) {
CarbonLoadModel copy = new CarbonLoadModel();
copy.tableName = tableName;
- copy.dimFolderPath = dimFolderPath;
copy.factFilePath = factFilePath + '/' + uniqueId;
copy.databaseName = databaseName;
copy.partitionId = uniqueId;
@@ -372,7 +356,6 @@ public class CarbonLoadModel implements Serializable {
String header, String delimiter) {
CarbonLoadModel copyObj = new CarbonLoadModel();
copyObj.tableName = tableName;
- copyObj.dimFolderPath = dimFolderPath;
copyObj.factFilePath = null;
copyObj.databaseName = databaseName;
copyObj.partitionId = uniqueId;
[2/2] incubator-carbondata git commit: [CARBONDATA-564]Remove unused
dimension table csv file to make dictionary This closes #467
Posted by ja...@apache.org.
[CARBONDATA-564]Remove unused dimension table csv file to make dictionary This closes #467
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/aaf98edc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/aaf98edc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/aaf98edc
Branch: refs/heads/master
Commit: aaf98edcaf290a6b1d6f2b5618ea9caf3fa61f4d
Parents: fd9d102 ff1bde1
Author: jackylk <ja...@huawei.com>
Authored: Thu Dec 29 15:41:26 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 29 15:41:26 2016 +0800
----------------------------------------------------------------------
.../carbondata/spark/load/CarbonLoaderUtil.java | 2 -
.../spark/util/GlobalDictionaryUtil.scala | 25 ---------
.../execution/command/carbonTableSchema.scala | 6 --
.../spark/util/AllDictionaryTestCase.scala | 6 +-
.../AutoHighCardinalityIdentifyTestCase.scala | 2 -
...GlobalDictionaryUtilConcurrentTestCase.scala | 4 +-
.../util/GlobalDictionaryUtilTestCase.scala | 8 +--
.../execution/command/carbonTableSchema.scala | 6 --
.../processing/csvload/DataGraphExecuter.java | 59 --------------------
.../dataprocessor/DataProcessTaskStatus.java | 13 -----
.../dataprocessor/IDataProcessStatus.java | 4 --
.../processing/model/CarbonLoadModel.java | 17 ------
12 files changed, 5 insertions(+), 147 deletions(-)
----------------------------------------------------------------------