You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by vi...@apache.org on 2016/09/03 11:41:46 UTC
[2/3] incubator-carbondata git commit: Problem: Code contains
currentRestructureNumber variable which is not used in the code now
Problem: Code contains currentRestructureNumber variable which is not used in the code now
Impact area: Data load flow
Fix: Remove the usage of currentRestructureNumber variable everywhere in the code
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5aedf544
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5aedf544
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5aedf544
Branch: refs/heads/master
Commit: 5aedf54437757e645bb360a913c424418c433b45
Parents: ec58755
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Sep 2 16:06:05 2016 +0530
Committer: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Committed: Sat Sep 3 17:09:33 2016 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 76 ---
.../hadoop/test/util/StoreCreator.java | 36 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 526 +------------------
.../spark/load/DeleteLoadFolders.java | 96 ----
.../carbondata/spark/util/LoadMetadataUtil.java | 50 --
.../spark/rdd/CarbonDataLoadRDD.scala | 33 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 61 +--
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 3 +-
.../processing/csvreaderstep/CsvInputMeta.java | 26 -
.../graphgenerator/GraphGenerator.java | 24 +-
.../configuration/GraphConfigurationInfo.java | 12 -
.../processing/mdkeygen/MDKeyGenStepMeta.java | 24 -
.../merger/step/CarbonSliceMergerStepMeta.java | 24 -
.../sortdatastep/SortKeyStepMeta.java | 25 -
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 17 -
15 files changed, 30 insertions(+), 1003 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 6f2bcc8..77d041c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -64,7 +64,6 @@ import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
@@ -148,67 +147,6 @@ public final class CarbonUtil {
}
/**
- * This method checks whether Restructure Folder exists or not
- * and if not exist then return the number with which folder need to created.
- *
- * @param baseStorePath -
- * baselocation where folder will be created.
- * @return counter
- * counter with which folder will be created.
- */
- public static int checkAndReturnCurrentRestructFolderNumber(String baseStorePath,
- final String filterType, final boolean isDirectory) {
- if (null == baseStorePath || 0 == baseStorePath.length()) {
- return -1;
- }
- // change the slashes to /
- baseStorePath = baseStorePath.replace("\\", "/");
-
- // check if string wnds with / then remove that.
- if (baseStorePath.charAt(baseStorePath.length() - 1) == '/') {
- baseStorePath = baseStorePath.substring(0, baseStorePath.lastIndexOf("/"));
- }
- int retValue = createBaseStoreFolders(baseStorePath);
- if (-1 == retValue) {
- return retValue;
- }
-
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(baseStorePath, FileFactory.getFileType(baseStorePath));
-
- // List of directories
- CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile pathname) {
- if (isDirectory && pathname.isDirectory()) {
- if (pathname.getAbsolutePath().indexOf(filterType) > -1) {
- return true;
- }
- } else {
- if (pathname.getAbsolutePath().indexOf(filterType) > -1) {
- return true;
- }
- }
-
- return false;
- }
- });
-
- int counter = -1;
-
- // if no folder exists then return -1
- if (listFiles.length == 0) {
- return counter;
- }
-
- counter = findCounterValue(filterType, listFiles, counter);
- return counter;
- }
-
- public static int checkAndReturnCurrentLoadFolderNumber(String baseStorePath) {
- return checkAndReturnCurrentRestructFolderNumber(baseStorePath, "Load_", true);
- }
-
- /**
* @param filterType
* @param listFiles
* @param counter
@@ -870,20 +808,6 @@ public final class CarbonUtil {
}
/**
- * @param location
- * @param factTableName
- * @return
- */
- public static int getRestructureNumber(String location, String factTableName) {
- String restructName =
- location.substring(location.indexOf(CarbonCommonConstants.RESTRUCTRE_FOLDER));
- int factTableIndex = restructName.indexOf(factTableName) - 1;
- String restructNumber =
- restructName.substring(CarbonCommonConstants.RESTRUCTRE_FOLDER.length(), factTableIndex);
- return Integer.parseInt(restructNumber);
- }
-
- /**
* Below method will be used to get the aggregator type
* CarbonCommonConstants.SUM_COUNT_VALUE_MEASURE will return when value is double measure
* CarbonCommonConstants.BYTE_VALUE_MEASURE will be returned when value is byte array
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 7960940..a48e6ad 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -18,21 +18,7 @@
*/
package org.apache.carbondata.hadoop.test.util;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
+import com.google.gson.Gson;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
@@ -83,7 +69,9 @@ import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-import com.google.gson.Gson;
+import java.io.*;
+import java.text.SimpleDateFormat;
+import java.util.*;
/**
* This class will create store file based on provided schema
@@ -124,7 +112,6 @@ public class StoreCreator {
absoluteTableIdentifier.getStorePath());
String kettleHomePath = "../processing/carbonplugins";
- int currentRestructureNumber = 0;
CarbonTable table = createTable();
writeDictionary(factFilePath, table);
CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
@@ -137,8 +124,7 @@ public class StoreCreator {
loadModel.setFactFilePath(factFilePath);
loadModel.setLoadMetadataDetails(new ArrayList<LoadMetadataDetails>());
- executeGraph(loadModel, absoluteTableIdentifier.getStorePath(), kettleHomePath,
- currentRestructureNumber);
+ executeGraph(loadModel, absoluteTableIdentifier.getStorePath(), kettleHomePath);
} catch (Exception e) {
e.printStackTrace();
}
@@ -327,11 +313,10 @@ public class StoreCreator {
* @param loadModel
* @param storeLocation
* @param kettleHomePath
- * @param currentRestructNumber
* @throws Exception
*/
- public static void executeGraph(LoadModel loadModel, String storeLocation, String kettleHomePath,
- int currentRestructNumber) throws Exception {
+ public static void executeGraph(LoadModel loadModel, String storeLocation, String kettleHomePath)
+ throws Exception {
System.setProperty("KETTLE_HOME", kettleHomePath);
new File(storeLocation).mkdirs();
String outPutLoc = storeLocation + "/etl";
@@ -372,7 +357,7 @@ public class StoreCreator {
info.setTableName(tableName);
generateGraph(schmaModel, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
- currentRestructNumber, loadModel.getLoadMetadataDetails());
+ loadModel.getLoadMetadataDetails());
DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
graphExecuter
@@ -460,13 +445,12 @@ public class StoreCreator {
* @param partitionID
* @param schema
* @param factStoreLocation
- * @param currentRestructNumber
* @param loadMetadataDetails
* @throws GraphGeneratorException
*/
private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
String tableName, String partitionID, CarbonDataLoadSchema schema, String factStoreLocation,
- int currentRestructNumber, List<LoadMetadataDetails> loadMetadataDetails)
+ List<LoadMetadataDetails> loadMetadataDetails)
throws GraphGeneratorException {
DataLoadModel model = new DataLoadModel();
model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
@@ -489,7 +473,7 @@ public class StoreCreator {
.getProperty("store_output_location", "../carbon-store/system/carbon/etl");
GraphGenerator generator =
new GraphGenerator(model, hdfsReadMode, partitionID, factStoreLocation,
- currentRestructNumber, allocate, schema, "0", outputLocation);
+ allocate, schema, "0", outputLocation);
generator.generateGraph();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 54c8f5e..246597c 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -19,7 +19,6 @@
package org.apache.carbondata.spark.load;
import java.io.BufferedWriter;
-import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
@@ -55,8 +54,6 @@ import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.carbon.path.CarbonStorePath;
import org.apache.carbondata.core.carbon.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -85,9 +82,6 @@ import org.apache.carbondata.spark.merger.NodeBlockRelation;
import org.apache.carbondata.spark.merger.NodeMultiBlockRelation;
import com.google.gson.Gson;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.util.Utils;
@@ -112,19 +106,12 @@ public final class CarbonLoaderUtil {
}
}
- /**
- * dfs.bytes-per-checksum
- * HDFS checksum length, block size for a file should be exactly divisible
- * by this value
- */
- private static final int HDFS_CHECKSUM_LENGTH = 512;
-
private CarbonLoaderUtil() {
}
private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
- int currentRestructNumber, CarbonLoadModel loadModel, String outputLocation)
+ CarbonLoadModel loadModel, String outputLocation)
throws GraphGeneratorException {
DataLoadModel model = new DataLoadModel();
model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
@@ -146,13 +133,13 @@ public final class CarbonLoaderUtil {
schmaModel.getCsvFilePath() != null && schmaModel.getCsvFilePath().startsWith("hdfs:");
int allocate = null != schmaModel.getCsvFilePath() ? 1 : schmaModel.getFilesToProcess().size();
GraphGenerator generator = new GraphGenerator(model, hdfsReadMode, loadModel.getPartitionId(),
- loadModel.getStorePath(), currentRestructNumber, allocate,
+ loadModel.getStorePath(), allocate,
loadModel.getCarbonDataLoadSchema(), loadModel.getSegmentId(), outputLocation);
generator.generateGraph();
}
public static void executeGraph(CarbonLoadModel loadModel, String storeLocation,
- String hdfsStoreLocation, String kettleHomePath, int currentRestructNumber) throws Exception {
+ String hdfsStoreLocation, String kettleHomePath) throws Exception {
System.setProperty("KETTLE_HOME", kettleHomePath);
if (!new File(storeLocation).mkdirs()) {
LOGGER.error("Error while creating the temp store path: " + storeLocation);
@@ -202,7 +189,7 @@ public final class CarbonLoaderUtil {
info.setComplexDelimiterLevel2(loadModel.getComplexDelimiterLevel2());
info.setSerializationNullFormat(loadModel.getSerializationNullFormat());
- generateGraph(schmaModel, info, currentRestructNumber, loadModel, outPutLoc);
+ generateGraph(schmaModel, info, loadModel, outPutLoc);
DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
graphExecuter
@@ -210,89 +197,11 @@ public final class CarbonLoaderUtil {
info, loadModel.getPartitionId(), loadModel.getCarbonDataLoadSchema());
}
- public static String[] getStorelocs(String databaseName, String tableName, String factTableName,
- String hdfsStoreLocation, int currentRestructNumber) {
- String[] loadFolders;
-
- String baseStorelocation =
- hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
- String factStorepath =
- baseStorelocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
- + currentRestructNumber + File.separator + factTableName;
-
- // Change to LOCAL for testing in local
- CarbonFile file =
- FileFactory.getCarbonFile(factStorepath, FileFactory.getFileType(factStorepath));
-
- if (!file.exists()) {
- return new String[0];
- }
- CarbonFile[] listFiles = file.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile path) {
- return path.getName().startsWith(CarbonCommonConstants.LOAD_FOLDER) && !path.getName()
- .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
- }
- });
-
- loadFolders = new String[listFiles.length];
- int count = 0;
-
- for (CarbonFile loadFile : listFiles) {
- loadFolders[count++] = loadFile.getAbsolutePath();
- }
-
- return loadFolders;
- }
-
public static List<String> addNewSliceNameToList(String newSlice, List<String> activeSlices) {
activeSlices.add(newSlice);
return activeSlices;
}
- public static String getAggLoadFolderLocation(String loadFolderName, String databaseName,
- String tableName, String aggTableName, String hdfsStoreLocation, int currentRestructNumber) {
- for (int i = currentRestructNumber; i >= 0; i--) {
- String aggTableLocation =
- getTableLocation(databaseName, tableName, aggTableName, hdfsStoreLocation, i);
- String aggStorepath = aggTableLocation + File.separator + loadFolderName;
- try {
- if (FileFactory.isFileExist(aggStorepath, FileFactory.getFileType(aggStorepath))) {
- return aggStorepath;
- }
- } catch (IOException e) {
- LOGGER.error("Problem checking file existence :: " + e.getMessage());
- }
- }
- return null;
- }
-
- public static String getTableLocation(String databaseName, String tableName, String aggTableName,
- String hdfsStoreLocation, int currentRestructNumber) {
- String baseStorelocation =
- hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
- String aggTableLocation =
- baseStorelocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
- + currentRestructNumber + File.separator + aggTableName;
- return aggTableLocation;
- }
-
- public static void deleteTable(int partitionCount, String databaseName, String tableName,
- String aggTableName, String hdfsStoreLocation, int currentRestructNumber) {
- String aggTableLoc = null;
- String partitionDatabaseName = null;
- String partitionTableName = null;
- for (int i = 0; i < partitionCount; i++) {
- partitionDatabaseName = databaseName + '_' + i;
- partitionTableName = tableName + '_' + i;
- for (int j = currentRestructNumber; j >= 0; j--) {
- aggTableLoc = getTableLocation(partitionDatabaseName, partitionTableName, aggTableName,
- hdfsStoreLocation, j);
- deleteStorePath(aggTableLoc);
- }
- }
- }
-
public static void deleteSegment(CarbonLoadModel loadModel, int currentLoad) {
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
CarbonTablePath carbonTablePath = CarbonStorePath
@@ -304,22 +213,6 @@ public final class CarbonLoaderUtil {
}
}
- public static void deleteSlice(int partitionCount, String databaseName, String tableName,
- String hdfsStoreLocation, int currentRestructNumber, String loadFolder) {
- String tableLoc = null;
- String partitionDatabaseName = null;
- String partitionTableName = null;
- for (int i = 0; i < partitionCount; i++) {
- partitionDatabaseName = databaseName + '_' + i;
- partitionTableName = tableName + '_' + i;
- tableLoc =
- getTableLocation(partitionDatabaseName, partitionTableName, tableName, hdfsStoreLocation,
- currentRestructNumber);
- tableLoc = tableLoc + File.separator + loadFolder;
- deleteStorePath(tableLoc);
- }
- }
-
public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel,
final boolean isCompactionFlow) throws IOException {
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
@@ -383,24 +276,6 @@ public final class CarbonLoaderUtil {
}
}
- public static boolean isSliceValid(String loc, List<String> activeSlices,
- List<String> updatedSlices, String factTableName) {
- String loadFolderName = loc.substring(loc.indexOf(CarbonCommonConstants.LOAD_FOLDER));
- String sliceNum = loadFolderName.substring(CarbonCommonConstants.LOAD_FOLDER.length());
- if (activeSlices.contains(loadFolderName) || updatedSlices.contains(sliceNum)) {
- String factFileLoc =
- loc + File.separator + factTableName + "_0" + CarbonCommonConstants.FACT_FILE_EXT;
- try {
- if (FileFactory.isFileExist(factFileLoc, FileFactory.getFileType(factFileLoc))) {
- return true;
- }
- } catch (IOException e) {
- LOGGER.error("Problem checking file existence :: " + e.getMessage());
- }
- }
- return false;
- }
-
public static List<String> getListOfValidSlices(LoadMetadataDetails[] details) {
List<String> activeSlices =
new ArrayList<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -439,24 +314,6 @@ public final class CarbonLoaderUtil {
// TODO: Remove from memory
}
- public static void createEmptyLoadFolder(CarbonLoadModel model, String factLoadFolderLocation,
- String hdfsStoreLocation, int currentRestructNumber) {
- String loadFolderName = factLoadFolderLocation
- .substring(factLoadFolderLocation.indexOf(CarbonCommonConstants.LOAD_FOLDER));
- String aggLoadFolderLocation =
- getTableLocation(model.getDatabaseName(), model.getTableName(), model.getAggTableName(),
- hdfsStoreLocation, currentRestructNumber);
- aggLoadFolderLocation = aggLoadFolderLocation + File.separator + loadFolderName;
- FileType fileType = FileFactory.getFileType(hdfsStoreLocation);
- try {
- FileFactory.mkdirs(aggLoadFolderLocation, fileType);
- } catch (IOException e) {
- LOGGER
- .error("Problem creating empty folder created for aggregation table: " + e.getMessage());
- }
- LOGGER.info("Empty folder created for aggregation table");
- }
-
/**
* This method will delete the local data load folder location after data load is complete
*
@@ -484,48 +341,6 @@ public final class CarbonLoaderUtil {
}
/**
- * This method will copy the current segment load to cgiven carbon store path
- *
- * @param loadModel
- * @param segmentName
- * @param updatedSlices
- * @throws IOException
- * @throws CarbonUtilException
- */
- public static void copyCurrentLoadToHDFS(CarbonLoadModel loadModel, String segmentName,
- List<String> updatedSlices) throws IOException, CarbonUtilException {
- //Copy the current load folder to carbon store
- boolean copyStore =
- Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
- String databaseName = loadModel.getDatabaseName();
- String tableName = loadModel.getTableName();
- String aggTableName = loadModel.getAggTableName();
- if (copyStore) {
- CarbonTableIdentifier carbonTableIdentifier =
- loadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();
- String segmentId = segmentName.substring(CarbonCommonConstants.LOAD_FOLDER.length());
- // form carbon store location
- String carbonStoreLocation = getStoreLocation(
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS),
- carbonTableIdentifier, segmentId, loadModel.getPartitionId());
- String tempLocationKey = databaseName + '_' + tableName;
- // form local store location
- String localStoreLocation = getStoreLocation(CarbonProperties.getInstance()
- .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL),
- carbonTableIdentifier, segmentId, loadModel.getPartitionId());
- localStoreLocation = localStoreLocation + File.separator + loadModel.getTaskNo();
- boolean isUpdate = false;
- if (loadModel.isAggLoadRequest() && null != aggTableName) {
- if (updatedSlices.contains(segmentId)) {
- isUpdate = true;
- }
- }
- copyToHDFS(carbonStoreLocation, localStoreLocation, isUpdate);
- CarbonUtil.deleteFoldersAndFiles(new File[] { new File(localStoreLocation) });
- }
- }
-
- /**
* This method will get the store location for the given path, segemnt id and partition id
*
* @param storePath
@@ -543,205 +358,6 @@ public final class CarbonLoaderUtil {
}
/**
- * This method will copy the carbon data files form local store segment folder
- * to carbon store segment location
- *
- * @param carbonStoreLocation
- * @param localStoreLocation
- * @param isUpdate
- * @throws IOException
- */
- private static void copyToHDFS(String carbonStoreLocation, String localStoreLocation,
- boolean isUpdate) throws IOException {
- //If the carbon store and the local store configured differently, then copy
- if (carbonStoreLocation != null && !carbonStoreLocation.equals(localStoreLocation)) {
- long copyStartTime = System.currentTimeMillis();
- // isUpdate will be true only when the below 2 conditions are satisfied
- // 1. slice is marked for update, 2. request is for aggregate table creation
- if (isUpdate) {
- renameFactFile(localStoreLocation);
- }
- LOGGER.info("Copying " + localStoreLocation + " --> " + carbonStoreLocation);
- CarbonUtil.checkAndCreateFolder(carbonStoreLocation);
- long blockSize = getBlockSize();
- int bufferSize = CarbonCommonConstants.BYTEBUFFER_SIZE;
- CarbonFile carbonFile = FileFactory
- .getCarbonFile(localStoreLocation, FileFactory.getFileType(localStoreLocation));
- CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile path) {
- return path.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) && !path.getName()
- .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
- }
- });
- for (int i = 0; i < listFiles.length; i++) {
- String localFilePath = listFiles[i].getCanonicalPath();
- CarbonFile localCarbonFile =
- FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
- String carbonFilePath = carbonStoreLocation + localFilePath
- .substring(localFilePath.lastIndexOf(File.separator));
- copyLocalFileToHDFS(carbonFilePath, localFilePath, bufferSize,
- getMaxOfBlockAndFileSize(blockSize, localCarbonFile.getSize()));
- }
- LOGGER.info("Total copy time (ms): " + (System.currentTimeMillis() - copyStartTime));
- } else {
- LOGGER.info("Separate carbon.storelocation.hdfs is not configured for carbon store path");
- }
- }
-
- /**
- * This method will return max of block size and file size
- *
- * @param blockSize
- * @param fileSize
- * @return
- */
- private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
- long maxSize = blockSize;
- if (fileSize > blockSize) {
- maxSize = fileSize;
- }
- // block size should be exactly divisible by 512 which is maintained by HDFS as bytes
- // per checksum, dfs.bytes-per-checksum=512 must divide block size
- long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
- if (remainder > 0) {
- maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
- }
- return maxSize;
- }
-
- /**
- * This method will return the block size for file to be copied in HDFS
- *
- * @return
- */
- private static long getBlockSize() {
- CarbonProperties carbonProperties = CarbonProperties.getInstance();
- long blockSizeInBytes = Long.parseLong(carbonProperties
- .getProperty(CarbonCommonConstants.MAX_FILE_SIZE,
- CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL))
- * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR
- * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR * 1L;
- return blockSizeInBytes;
- }
-
- /**
- * This method will read the local carbon data file and write to carbon data file in HDFS
- *
- * @param carbonStoreFilePath
- * @param localFilePath
- * @param bufferSize
- * @param blockSize
- * @throws IOException
- */
- private static void copyLocalFileToHDFS(String carbonStoreFilePath, String localFilePath,
- int bufferSize, long blockSize) throws IOException {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- try {
- LOGGER.debug(
- "HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize + " (bytes");
- dataOutputStream = FileFactory
- .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
- bufferSize, blockSize);
- dataInputStream = FileFactory
- .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
- IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
- } finally {
- CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
- }
- }
-
- private static void renameFactFile(String localStoreLocation) {
- FileType fileType = FileFactory.getFileType(localStoreLocation);
- try {
- if (FileFactory.isFileExist(localStoreLocation, fileType)) {
- CarbonFile carbonFile = FileFactory.getCarbonFile(localStoreLocation, fileType);
- CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile path) {
- return path.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) && !path.getName()
- .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
- }
- });
- for (int i = 0; i < listFiles.length; i++) {
- carbonFile = listFiles[i];
- String factFilePath = carbonFile.getCanonicalPath();
- String changedFileName = factFilePath.replace(CarbonCommonConstants.FACT_FILE_EXT,
- CarbonCommonConstants.FACT_UPDATE_EXTENSION);
- carbonFile.renameTo(changedFileName);
- }
- }
- } catch (IOException e) {
- LOGGER.error("Inside renameFactFile. Problem checking file existence :: " + e.getMessage());
- }
- }
-
- /**
- * API will provide the load number inorder to record the same in metadata file.
- */
- public static int getLoadCount(CarbonLoadModel loadModel, int currentRestructNumber)
- throws IOException {
-
- String hdfsLoadedTable = getLoadFolderPath(loadModel, null, null, currentRestructNumber);
- int loadCounter = CarbonUtil.checkAndReturnCurrentLoadFolderNumber(hdfsLoadedTable);
-
- String hdfsStoreLoadFolder =
- hdfsLoadedTable + File.separator + CarbonCommonConstants.LOAD_FOLDER + loadCounter;
- hdfsStoreLoadFolder = hdfsStoreLoadFolder.replace("\\", "/");
-
- String loadFolerCount = hdfsStoreLoadFolder
- .substring(hdfsStoreLoadFolder.lastIndexOf('_') + 1, hdfsStoreLoadFolder.length());
- return Integer.parseInt(loadFolerCount) + 1;
- }
-
- /**
- * API will provide the load folder path for the store inorder to store the same
- * in the metadata.
- */
- public static String getLoadFolderPath(CarbonLoadModel loadModel, String tableName,
- String databaseName, int currentRestructNumber) {
-
- //CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005
-
- boolean copyStore =
- Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
-
- // CHECKSTYLE:ON
- if (null == tableName && null == databaseName) {
- databaseName = loadModel.getDatabaseName();
- tableName = loadModel.getTableName();
- }
- String factTable = loadModel.getTableName();
- String hdfsLoadedTable = null;
- if (copyStore) {
- String hdfsLocation =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
- String localStore = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.STORE_LOCATION,
- CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
-
- if (!hdfsLocation.equals(localStore)) {
- String hdfsStoreLocation = hdfsLocation;
- hdfsStoreLocation =
- hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
- int rsCounter = currentRestructNumber;
- if (rsCounter == -1) {
- rsCounter = 0;
- }
-
- hdfsLoadedTable =
- hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
- + File.separator + factTable;
-
- hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
- }
-
- }
- return hdfsLoadedTable;
-
- }
-
- /**
* This API will write the load level metadata for the loadmanagement module inorder to
* manage the load and query execution management smoothly.
*
@@ -875,140 +491,6 @@ public final class CarbonLoaderUtil {
return carbonTable.getMetaDataFilepath();
}
- /**
- * This method will provide the dimension column list for a given aggregate
- * table
- */
- public static Set<String> getColumnListFromAggTable(CarbonLoadModel model) {
- Set<String> columnList = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- CarbonTable carbonTable =
- org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
- .getCarbonTable(model.getDatabaseName() + '_' + model.getTableName());
- List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(model.getAggTableName());
- List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(model.getAggTableName());
- for (CarbonDimension carbonDimension : dimensions) {
- columnList.add(carbonDimension.getColName());
- }
- for (CarbonMeasure carbonMeasure : measures) {
- columnList.add(carbonMeasure.getColName());
- }
- return columnList;
- }
-
- public static void copyMergedLoadToHDFS(CarbonLoadModel loadModel, int currentRestructNumber,
- String mergedLoadName) {
- //Copy the current load folder to HDFS
- boolean copyStore =
- Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
-
- String databaseName = loadModel.getDatabaseName();
- String tableName = loadModel.getTableName();
- String factTable = loadModel.getTableName();
- String aggTableName = loadModel.getAggTableName();
-
- if (copyStore) {
- String hdfsLocation =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
-
- String localStore = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.STORE_LOCATION,
- CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
- if (!loadModel.isAggLoadRequest()) {
- copyMergeToHDFS(databaseName, tableName, factTable, hdfsLocation, localStore,
- currentRestructNumber, mergedLoadName);
- }
- if (null != aggTableName) {
- copyMergeToHDFS(databaseName, tableName, aggTableName, hdfsLocation, localStore,
- currentRestructNumber, mergedLoadName);
- }
- try {
- CarbonUtil.deleteFoldersAndFiles(new File[] {
- new File(localStore + File.separator + databaseName + File.separator + tableName) });
- } catch (CarbonUtilException e) {
- LOGGER.error("Error while CarbonUtil.deleteFoldersAndFiles ");
- }
- }
- }
-
- public static void copyMergeToHDFS(String databaseName, String tableName, String factTable,
- String hdfsLocation, String localStore, int currentRestructNumber, String mergedLoadName) {
- try {
- //If the hdfs store and the local store configured differently, then copy
- if (hdfsLocation != null && !hdfsLocation.equals(localStore)) {
- /**
- * Identify the Load_X folder from the local store folder
- */
- String currentloadedStore = localStore;
- currentloadedStore =
- currentloadedStore + File.separator + databaseName + File.separator + tableName;
-
- int rsCounter = currentRestructNumber;
-
- if (rsCounter == -1) {
- LOGGER.info("Unable to find the local store details (RS_-1) " + currentloadedStore);
- return;
- }
- String localLoadedTable =
- currentloadedStore + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
- + rsCounter + File.separator + factTable;
-
- localLoadedTable = localLoadedTable.replace("\\", "/");
-
- int loadCounter = CarbonUtil.checkAndReturnCurrentLoadFolderNumber(localLoadedTable);
-
- if (loadCounter == -1) {
- LOGGER.info("Unable to find the local store details (Load_-1) " + currentloadedStore);
-
- return;
- }
-
- String localLoadName = CarbonCommonConstants.LOAD_FOLDER + mergedLoadName;
- String localLoadFolder =
- localLoadedTable + File.separator + CarbonCommonConstants.LOAD_FOLDER + mergedLoadName;
-
- LOGGER.info("Local data loaded folder ... = " + localLoadFolder);
-
- //Identify the Load_X folder in the HDFS store
- String hdfsStoreLocation = hdfsLocation;
- hdfsStoreLocation =
- hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
- rsCounter = currentRestructNumber;
- if (rsCounter == -1) {
- rsCounter = 0;
- }
-
- String hdfsLoadedTable =
- hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
- + File.separator + factTable;
-
- hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
-
- String hdfsStoreLoadFolder = hdfsLoadedTable + File.separator + localLoadName;
-
- LOGGER.info("HDFS data load folder ... = " + hdfsStoreLoadFolder);
-
- // Copy the data created through latest ETL run, to the HDFS store
- LOGGER.info("Copying " + localLoadFolder + " --> " + hdfsStoreLoadFolder);
-
- hdfsStoreLoadFolder = hdfsStoreLoadFolder.replace("\\", "/");
- Path path = new Path(hdfsStoreLocation);
-
- FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- fs.copyFromLocalFile(true, true, new Path(localLoadFolder), new Path(hdfsStoreLoadFolder));
-
- LOGGER.info("Copying sliceMetaData from " + localLoadedTable + " --> " + hdfsLoadedTable);
-
- } else {
- LOGGER.info("Separate carbon.storelocation.hdfs is not configured for hdfs store path");
- }
- } catch (RuntimeException e) {
- LOGGER.info(e.getMessage());
- } catch (Exception e) {
- LOGGER.info(e.getMessage());
- }
- }
-
public static Dictionary getDictionary(DictionaryColumnUniqueIdentifier columnIdentifier,
String carbonStorePath) throws CarbonUtilException {
Cache dictCache =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index d2af4f9..b9ac4dd 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -45,7 +45,6 @@ import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.spark.util.LoadMetadataUtil;
public final class DeleteLoadFolders {
@@ -56,67 +55,6 @@ public final class DeleteLoadFolders {
}
- public static boolean deleteLoadFoldersFromFileSystem(CarbonLoadModel loadModel,
- int partitionCount, String storeLocation, boolean isForceDelete, int currentRestructNumber,
- LoadMetadataDetails[] details) {
- String path = null;
- List<LoadMetadataDetails> deletedLoads =
- new ArrayList<LoadMetadataDetails>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- boolean isDeleted = false;
-
- if (details != null && details.length != 0) {
- for (LoadMetadataDetails oneLoad : details) {
- if (checkIfLoadCanBeDeleted(oneLoad, isForceDelete)) {
- boolean deletionStatus = false;
-
- for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
- // check load folder in each restructure folder
- for (int restructureFolderNum = 0;
- restructureFolderNum <= currentRestructNumber; restructureFolderNum++) {
- CarbonFile[] aggFiles = LoadMetadataUtil
- .getAggregateTableList(loadModel, storeLocation, partitionId,
- restructureFolderNum);
- deleteAggLoadFolders(aggFiles,
- CarbonCommonConstants.LOAD_FOLDER + oneLoad.getLoadName());
- path = LoadMetadataUtil.createLoadFolderPath(loadModel, storeLocation, partitionId,
- restructureFolderNum);
- String loadFolderPath = "";
- // deleting merged load folder
- if (oneLoad.getMergedLoadName() != null) {
- loadFolderPath =
- path + CarbonCommonConstants.FILE_SEPARATOR + CarbonCommonConstants.LOAD_FOLDER
- + oneLoad.getMergedLoadName();
- deletionStatus = physicalFactAndMeasureMetadataDeletion(loadFolderPath);
- } else {
- loadFolderPath =
- path + CarbonCommonConstants.FILE_SEPARATOR + CarbonCommonConstants.LOAD_FOLDER
- + oneLoad.getLoadName();
- deletionStatus = physicalFactAndMeasureMetadataDeletion(loadFolderPath);
- }
- if (deletionStatus) {
- cleanDeletedFactFile(loadFolderPath);
- factFileRenaming(loadFolderPath);
- // if deletion status is True then there is no
- // need to traverse all the RS folders.
- break;
- }
- }
-
- }
- if (deletionStatus) {
- isDeleted = true;
- oneLoad.setVisibility("false");
- deletedLoads.add(oneLoad);
- LOGGER.info(" Deleted the load " + oneLoad.getLoadName());
- }
- }
- }
- }
-
- return isDeleted;
- }
-
/**
* returns segment path
*
@@ -138,40 +76,6 @@ public final class DeleteLoadFolders {
return path;
}
- public static void deleteAggLoadFolders(CarbonFile[] aggFiles, String loadName) {
- for (CarbonFile file : aggFiles) {
- deleteLoadFolderFromEachAgg(file, loadName);
- }
-
- }
-
- private static void deleteLoadFolderFromEachAgg(CarbonFile file, final String loadName) {
- CarbonFile[] loadFolders = file.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile file) {
- if (file.getName().equalsIgnoreCase(loadName)) {
- return true;
- }
- return false;
- }
- });
-
- for (CarbonFile loadFolder : loadFolders) {
- CarbonFile[] files = loadFolder.listFiles();
- // deleting individual files
- if (files != null) {
- for (CarbonFile eachFile : files) {
- if (!eachFile.delete()) {
- LOGGER.warn("Unable to delete the file as per delete command "
- + loadFolder.getAbsolutePath());
- }
- }
- }
-
- }
-
- }
-
private static boolean physicalFactAndMeasureMetadataDeletion(String path) {
boolean status = false;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
index 6110fb8..a897e80 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
@@ -28,13 +28,8 @@
*/
package org.apache.carbondata.spark.util;
-import java.io.File;
-
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.lcm.status.SegmentStatusManager;
import org.apache.carbondata.spark.load.CarbonLoadModel;
@@ -65,49 +60,4 @@ public final class LoadMetadataUtil {
return false;
}
-
- public static String createLoadFolderPath(CarbonLoadModel model, String hdfsStoreLocation,
- int partitionId, int currentRestructNumber) {
- hdfsStoreLocation =
- hdfsStoreLocation + File.separator + model.getDatabaseName() + '_' + partitionId
- + File.separator + model.getTableName() + '_' + partitionId;
- int rsCounter = currentRestructNumber;
- if (rsCounter == -1) {
- rsCounter = 0;
- }
- String hdfsLoadedTable =
- hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
- + File.separator + model.getTableName();
- hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
- return hdfsLoadedTable;
- }
-
- public static CarbonFile[] getAggregateTableList(final CarbonLoadModel model,
- String hdfsStoreLocation, int partitionId, int currentRestructNumber) {
- hdfsStoreLocation =
- hdfsStoreLocation + File.separator + model.getDatabaseName() + '_' + partitionId
- + File.separator + model.getTableName() + '_' + partitionId;
-
- int rsCounter = currentRestructNumber;
- if (rsCounter == -1) {
- rsCounter = 0;
- }
-
- String hdfsLoadedTable =
- hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter;
-
- CarbonFile rsFile =
- FileFactory.getCarbonFile(hdfsLoadedTable, FileFactory.getFileType(hdfsLoadedTable));
-
- CarbonFile[] aggFiles = rsFile.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(
- CarbonCommonConstants.AGGREGATE_TABLE_START_TAG + CarbonCommonConstants.UNDERSCORE
- + model.getTableName());
- }
- });
-
- return aggFiles;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index abbca33..60a2d00 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -83,7 +83,6 @@ class CarbonNodePartition(rddId: Int, val idx: Int, host: String,
* @param kettleHomePath The kettle home path
* @param partitioner Partitioner which specify how to partition
* @param columinar whether it is columinar
- * @param currentRestructNumber current restruct number
* @param loadCount Current load count
* @param tableCreationTime Time of creating table
* @param schemaLastUpdatedTime Time of last schema update
@@ -101,7 +100,6 @@ class CarbonDataLoadRDD[K, V](
kettleHomePath: String,
partitioner: Partitioner,
columinar: Boolean,
- currentRestructNumber: Integer,
loadCount: Integer,
tableCreationTime: Long,
schemaLastUpdatedTime: Long,
@@ -198,8 +196,7 @@ class CarbonDataLoadRDD[K, V](
}
else {
try {
- CarbonLoaderUtil.executeGraph(model, storeLocation, hdfsStoreLocation, kettleHomePath,
- currentRestructNumber)
+ CarbonLoaderUtil.executeGraph(model, storeLocation, hdfsStoreLocation, kettleHomePath)
} catch {
case e: DataLoadingException => if (e.getErrorCode ==
DataProcessorConstants.BAD_REC_FOUND) {
@@ -324,13 +321,7 @@ class CarbonDataLoadRDD[K, V](
val copyListOfLoadFolders = listOfLoadFolders.asScala.toList
val copyListOfUpdatedLoadFolders = listOfUpdatedLoadFolders.asScala.toList
loadTableSlices(listOfAllLoadFolders, details)
- var loadFolders = Array[String]()
- val loadFolder = CarbonLoaderUtil
- .getAggLoadFolderLocation(newSlice, model.getDatabaseName, model.getTableName,
- model.getTableName, hdfsStoreLocation, currentRestructNumber)
- if (null != loadFolder) {
- loadFolders :+= loadFolder
- }
+ val loadFolders = Array[String]()
dataloadStatus = iterateOverAggTables(aggTables, copyListOfLoadFolders.asJava,
copyListOfUpdatedLoadFolders.asJava, loadFolders)
if (CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(dataloadStatus)) {
@@ -358,14 +349,7 @@ class CarbonDataLoadRDD[K, V](
val listOfLoadFolders = CarbonLoaderUtil.getListOfValidSlices(details)
val listOfUpdatedLoadFolders = CarbonLoaderUtil.getListOfUpdatedSlices(details)
loadTableSlices(listOfAllLoadFolders, details)
- var loadFolders = Array[String]()
- var restructFolders = Array[String]()
- for (number <- 0 to currentRestructNumber) {
- restructFolders = CarbonLoaderUtil
- .getStorelocs(model.getDatabaseName, model.getTableName, model.getTableName,
- hdfsStoreLocation, number)
- loadFolders = loadFolders ++ restructFolders
- }
+ val loadFolders = Array[String]()
val aggTable = model.getAggTableName
dataloadStatus = loadAggregationTable(listOfLoadFolders, listOfUpdatedLoadFolders,
loadFolders)
@@ -386,16 +370,7 @@ class CarbonDataLoadRDD[K, V](
val listOfUpdatedLoadFolders = CarbonLoaderUtil.getListOfUpdatedSlices(details)
val listOfAllLoadFolder = CarbonQueryUtil.getListOfSlices(details)
loadTableSlices(listOfAllLoadFolder, details)
- var loadFolders = Array[String]()
- listOfUpdatedLoadFolders.asScala.foreach { sliceNum =>
- val newSlice = CarbonCommonConstants.LOAD_FOLDER + sliceNum
- val loadFolder = CarbonLoaderUtil
- .getAggLoadFolderLocation(newSlice, model.getDatabaseName, model.getTableName,
- model.getTableName, hdfsStoreLocation, currentRestructNumber)
- if (null != loadFolder) {
- loadFolders :+= loadFolder
- }
- }
+ val loadFolders = Array[String]()
iterateOverAggTables(aggTables, listOfLoadFolders, listOfUpdatedLoadFolders, loadFolders)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index bf4fa9f..0d0ac97 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.carbon.metadata.CarbonMetadata
import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.integration.spark.merger.{CarbonCompactionUtil, CompactionCallable, CompactionType}
import org.apache.carbondata.lcm.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.lcm.status.SegmentStatusManager
@@ -102,11 +102,6 @@ object CarbonDataRDDFactory extends Logging {
val table = CarbonMetadata.getInstance()
.getCarbonTable(carbonLoadModel.getDatabaseName + "_" + carbonLoadModel.getTableName)
val metaDataPath: String = table.getMetaDataFilepath
- var currentRestructNumber = CarbonUtil
- .checkAndReturnCurrentRestructFolderNumber(metaDataPath, "RS_", false)
- if (-1 == currentRestructNumber) {
- currentRestructNumber = 0
- }
}
def deleteLoadByDate(
@@ -124,12 +119,6 @@ object CarbonDataRDDFactory extends Logging {
// Delete the records based on data
val table = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
.getCarbonTable(databaseName + "_" + tableName)
-
- var currentRestructNumber = CarbonUtil
- .checkAndReturnCurrentRestructFolderNumber(table.getMetaDataFilepath, "RS_", false)
- if (-1 == currentRestructNumber) {
- currentRestructNumber = 0
- }
val segmentStatusManager = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
val loadMetadataDetailsArray =
segmentStatusManager.readLoadMetadata(table.getMetaDataFilepath()).toList
@@ -145,8 +134,7 @@ object CarbonDataRDDFactory extends Logging {
table.getFactTableName,
tableName,
hdfsStoreLocation,
- loadMetadataDetailsArray,
- currentRestructNumber).collect.groupBy(_._1)
+ loadMetadataDetailsArray).collect.groupBy(_._1)
var updatedLoadMetadataDetailsList = new ListBuffer[LoadMetadataDetails]()
if (resultMap.nonEmpty) {
@@ -688,7 +676,6 @@ object CarbonDataRDDFactory extends Logging {
isAgg: Boolean,
partitionStatus: String = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS) {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- var currentRestructNumber = -1
// for handling of the segment Merging.
def handleSegmentMerging(tableCreationTime: Long): Unit = {
@@ -780,17 +767,9 @@ object CarbonDataRDDFactory extends Logging {
.audit("Data load request has been received for table " + carbonLoadModel
.getDatabaseName + "." + carbonLoadModel.getTableName
)
-
- currentRestructNumber = CarbonUtil
- .checkAndReturnCurrentRestructFolderNumber(carbonTable.getMetaDataFilepath, "RS_", false)
- if (-1 == currentRestructNumber) {
- currentRestructNumber = 0
- }
-
// Check if any load need to be deleted before loading new data
deleteLoadsAndUpdateMetadata(carbonLoadModel, carbonTable, partitioner, hdfsStoreLocation,
- isForceDeletion = false,
- currentRestructNumber)
+ isForceDeletion = false)
if (null == carbonLoadModel.getLoadMetadataDetails) {
readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
}
@@ -988,7 +967,6 @@ object CarbonDataRDDFactory extends Logging {
kettleHomePath,
partitioner,
columinar,
- currentRestructNumber,
currentLoadCount,
tableCreationTime,
schemaLastUpdatedTime,
@@ -1043,27 +1021,7 @@ object CarbonDataRDDFactory extends Logging {
if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
logInfo("********starting clean up**********")
- if (isAgg) {
- // TODO:need to clean aggTable
- CarbonLoaderUtil.deleteTable(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName, carbonLoadModel.getAggTableName, hdfsStoreLocation,
- currentRestructNumber
- )
- errorMessage = "Aggregate table creation failure"
- } else {
- CarbonLoaderUtil.deleteSegment(carbonLoadModel, currentLoadCount)
- val aggTables = carbonTable.getAggregateTablesName
- if (null != aggTables && !aggTables.isEmpty) {
- // TODO:need to clean aggTable
- val (result, _) = status(0)
- val newSlice = CarbonCommonConstants.LOAD_FOLDER + result
- aggTables.asScala.foreach { aggTableName =>
- CarbonLoaderUtil
- .deleteSlice(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName, hdfsStoreLocation, currentRestructNumber, newSlice)
- }
- }
- }
+ CarbonLoaderUtil.deleteSegment(carbonLoadModel, currentLoadCount)
logInfo("********clean up done**********")
logger.audit(s"Data load is failed for " +
s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
@@ -1110,8 +1068,7 @@ object CarbonDataRDDFactory extends Logging {
carbonLoadModel: CarbonLoadModel,
table: CarbonTable, partitioner: Partitioner,
hdfsStoreLocation: String,
- isForceDeletion: Boolean,
- currentRestructNumber: Integer) {
+ isForceDeletion: Boolean) {
if (LoadMetadataUtil.isLoadDeletionRequired(carbonLoadModel)) {
val loadMetadataFilePath = CarbonLoaderUtil
.extractLoadMetadataFileLocation(carbonLoadModel)
@@ -1152,11 +1109,6 @@ object CarbonDataRDDFactory extends Logging {
val table = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
.getCarbonTable(carbonLoadModel.getDatabaseName + "_" + carbonLoadModel.getTableName)
val metaDataPath: String = table.getMetaDataFilepath
- var currentRestructNumber = CarbonUtil
- .checkAndReturnCurrentRestructFolderNumber(metaDataPath, "RS_", false)
- if (-1 == currentRestructNumber) {
- currentRestructNumber = 0
- }
val carbonLock = CarbonLockFactory
.getCarbonLockObj(table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
LockUsage.METADATA_LOCK
@@ -1167,8 +1119,7 @@ object CarbonDataRDDFactory extends Logging {
table,
partitioner,
hdfsStoreLocation,
- isForceDeletion = true,
- currentRestructNumber)
+ isForceDeletion = true)
}
}
finally {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
index ed3be91..f3308d6 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
@@ -41,8 +41,7 @@ class CarbonDeleteLoadByDateRDD[K, V](
factTableName: String,
dimTableName: String,
hdfsStoreLocation: String,
- loadMetadataDetails: List[LoadMetadataDetails],
- currentRestructFolder: Integer)
+ loadMetadataDetails: List[LoadMetadataDetails])
extends RDD[(K, V)](sc, Nil) with Logging {
sc.setLocalProperty("spark.scheduler.pool", "DDL")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
index b15862d..2e0dece 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
@@ -93,8 +93,6 @@ public class CsvInputMeta extends BaseStepMeta
private boolean newlinePossibleInFields;
- private int currentRestructNumber;
-
private String blocksID;
private String partitionID;
@@ -120,7 +118,6 @@ public class CsvInputMeta extends BaseStepMeta
lazyConversionActive = true;
isaddresult = false;
bufferSize = "50000";
- currentRestructNumber = -1;
blocksID = "";
partitionID = "";
escapeCharacter ="\\";
@@ -157,8 +154,6 @@ public class CsvInputMeta extends BaseStepMeta
newlinePossibleInFields = "Y".equalsIgnoreCase(nlp);
}
encoding = XMLHandler.getTagValue(stepnode, getXmlCode("ENCODING"));
- currentRestructNumber =
- Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
blocksID = XMLHandler.getTagValue(stepnode, "blocksID");
partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
escapeCharacter = XMLHandler.getTagValue(stepnode, "escapeCharacter");
@@ -222,8 +217,6 @@ public class CsvInputMeta extends BaseStepMeta
retval.append(" ")
.append(XMLHandler.addTagValue(getXmlCode("NEWLINE_POSSIBLE"), newlinePossibleInFields));
retval.append(" ").append(XMLHandler.addTagValue(getXmlCode("ENCODING"), encoding));
- retval.append(" ")
- .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
retval.append(" ").append(XMLHandler.addTagValue("blocksID", blocksID));
retval.append(" ").append(XMLHandler.addTagValue("partitionID", partitionID));
retval.append(" ").append(XMLHandler.addTagValue("escapeCharacter", escapeCharacter));
@@ -277,7 +270,6 @@ public class CsvInputMeta extends BaseStepMeta
rep.getStepAttributeBoolean(idStep, 0, getRepCode("NEWLINE_POSSIBLE"),
!runningInParallel);
encoding = rep.getStepAttributeString(idStep, getRepCode("ENCODING"));
- currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
blocksID = rep.getStepAttributeString(idStep, getRepCode("blocksID"));
partitionID = rep.getStepAttributeString(idStep, getRepCode("partitionID"));
escapeCharacter = rep.getStepAttributeString(idStep, getRepCode("escapeCharacter"));
@@ -333,8 +325,6 @@ public class CsvInputMeta extends BaseStepMeta
rep.saveStepAttribute(idTransformation, idStep, getRepCode("NEWLINE_POSSIBLE"),
newlinePossibleInFields);
rep.saveStepAttribute(idTransformation, idStep, getRepCode("ENCODING"), encoding);
- rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
- currentRestructNumber);
rep.saveStepAttribute(idTransformation, idStep, getRepCode("blocksID"), blocksID);
rep.saveStepAttribute(idTransformation, idStep, getRepCode("partitionID"), partitionID);
rep.saveStepAttribute(idTransformation, idStep, getRepCode("escapeCharacter"),
@@ -832,8 +822,6 @@ public class CsvInputMeta extends BaseStepMeta
isaddresult = (Boolean) entry.getValue();
} else if ("ENCODING".equals(attributeKey)) {
encoding = (String) entry.getValue();
- } else if ("currentRestructNumber".equals(attributeKey)) {
- currentRestructNumber = (Integer) entry.getValue();
} else if ("blocksID".equals(attributeKey)) {
blocksID = (String) entry.getValue();
} else if ("partitionID".equals(attributeKey)) {
@@ -920,20 +908,6 @@ public class CsvInputMeta extends BaseStepMeta
this.newlinePossibleInFields = newlinePossibleInFields;
}
- /**
- * @return the currentRestructNumber
- */
- public int getCurrentRestructNumber() {
- return currentRestructNumber;
- }
-
- /**
- * @param currentRestructNum the currentRestructNumber to set
- */
- public void setCurrentRestructNumber(int currentRestructNum) {
- this.currentRestructNumber = currentRestructNum;
- }
-
public void setPartitionID(String partitionID) {
this.partitionID = partitionID;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index 10ea710..f55c247 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -46,7 +46,6 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeas
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.load.BlockDetails;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.csvreaderstep.CsvInputMeta;
@@ -184,7 +183,6 @@ public class GraphGenerator {
private boolean isColumnar;
private String factTableName;
private String factStoreLocation;
- private int currentRestructNumber;
private String blocksID;
private String escapeCharacter;
/**
@@ -205,7 +203,7 @@ public class GraphGenerator {
private String maxColumns;
public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID,
- String factStoreLocation, int currentRestructNum, int allocate,
+ String factStoreLocation, int allocate,
CarbonDataLoadSchema carbonDataLoadSchema, String segmentId) {
CarbonMetadata.getInstance().addCarbonTable(carbonDataLoadSchema.getCarbonTable());
this.schemaInfo = dataLoadModel.getSchemaInfo();
@@ -219,7 +217,6 @@ public class GraphGenerator {
this.partitionID = partitionID;
this.factStoreLocation = factStoreLocation;
this.isColumnar = Boolean.parseBoolean(CarbonCommonConstants.IS_COLUMNAR_STORAGE_DEFAULTVALUE);
- this.currentRestructNumber = currentRestructNum;
this.blocksID = dataLoadModel.getBlocksID();
this.taskNo = dataLoadModel.getTaskNo();
this.factTimeStamp = dataLoadModel.getFactTimeStamp();
@@ -231,10 +228,10 @@ public class GraphGenerator {
}
public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID,
- String factStoreLocation, int currentRestructNum, int allocate,
- CarbonDataLoadSchema carbonDataLoadSchema, String segmentId, String outputLocation) {
- this(dataLoadModel, isHDFSReadMode, partitionID, factStoreLocation, currentRestructNum,
- allocate, carbonDataLoadSchema, segmentId);
+ String factStoreLocation, int allocate, CarbonDataLoadSchema carbonDataLoadSchema,
+ String segmentId, String outputLocation) {
+ this(dataLoadModel, isHDFSReadMode, partitionID, factStoreLocation, allocate,
+ carbonDataLoadSchema, segmentId);
this.outputLocation = outputLocation;
}
@@ -442,7 +439,6 @@ public class GraphGenerator {
csvInputMeta.setEnclosure("\"");
csvInputMeta.setHeaderPresent(true);
csvInputMeta.setMaxColumns(maxColumns);
- csvInputMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
StepMeta csvDataStep =
new StepMeta(GraphGeneratorConstants.CSV_INPUT, (StepMetaInterface) csvInputMeta);
csvDataStep.setLocation(100, 100);
@@ -473,12 +469,6 @@ public class GraphGenerator {
sliceMerger.setTabelName(configurationInfo.getTableName());
sliceMerger.setTableName(schemaInfo.getTableName());
sliceMerger.setDatabaseName(schemaInfo.getDatabaseName());
- if (null != this.factStoreLocation) {
- sliceMerger.setCurrentRestructNumber(
- CarbonUtil.getRestructureNumber(this.factStoreLocation, this.factTableName));
- } else {
- sliceMerger.setCurrentRestructNumber(configurationInfo.getCurrentRestructNumber());
- }
sliceMerger.setGroupByEnabled(isAutoAggRequest + "");
if (isAutoAggRequest) {
String[] aggType = configurationInfo.getAggType();
@@ -559,7 +549,6 @@ public class GraphGenerator {
seqMeta.setDatabaseName(schemaInfo.getDatabaseName());
seqMeta.setComplexDelimiterLevel1(schemaInfo.getComplexDelimiterLevel1());
seqMeta.setComplexDelimiterLevel2(schemaInfo.getComplexDelimiterLevel2());
- seqMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
seqMeta.setCarbonMetaHier(graphConfiguration.getMetaHeirString());
seqMeta.setCarbonmsr(graphConfiguration.getMeasuresString());
seqMeta.setCarbonProps(graphConfiguration.getPropertiesString());
@@ -620,7 +609,6 @@ public class GraphGenerator {
carbonMdKey.setDatabaseName(schemaInfo.getDatabaseName());
carbonMdKey.setTableName(schemaInfo.getTableName());
carbonMdKey.setComplexTypeString(graphConfiguration.getComplexTypeString());
- carbonMdKey.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
carbonMdKey.setAggregateLevels(CarbonDataProcessorUtil
.getLevelCardinalitiesString(graphConfiguration.getDimCardinalities(),
graphConfiguration.getDimensions()));
@@ -765,7 +753,6 @@ public class GraphGenerator {
sortRowsMeta.setTableName(schemaInfo.getTableName());
sortRowsMeta.setDatabaseName(schemaInfo.getDatabaseName());
sortRowsMeta.setOutputRowSize(actualMeasures.length + 1 + "");
- sortRowsMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
sortRowsMeta.setDimensionCount(graphConfiguration.getDimensions().length + "");
sortRowsMeta.setComplexDimensionCount(graphConfiguration.getComplexTypeString().isEmpty() ?
"0" :
@@ -797,7 +784,6 @@ public class GraphGenerator {
CarbonDataLoadSchema carbonDataLoadSchema) throws GraphGeneratorException {
//
GraphConfigurationInfo graphConfiguration = new GraphConfigurationInfo();
- graphConfiguration.setCurrentRestructNumber(currentRestructNumber);
List<CarbonDimension> dimensions = carbonDataLoadSchema.getCarbonTable()
.getDimensionByTableName(carbonDataLoadSchema.getCarbonTable().getFactTableName());
prepareIsUseInvertedIndex(dimensions, graphConfiguration);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
index 6ff1eb1..7c428a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
@@ -186,7 +186,6 @@ public class GraphConfigurationInfo {
* type
*/
private char[] type;
- private int currentRestructNumber;
private String levelAnddataType;
private Boolean[] isNoDictionaryDimMapping;
@@ -950,17 +949,6 @@ public class GraphConfigurationInfo {
this.type = type;
}
- /**
- * @return currentRestructNumber the current Restruct Number
- */
- public int getCurrentRestructNumber() {
- return currentRestructNumber;
- }
-
- public void setCurrentRestructNumber(int currentRestructNum) {
- this.currentRestructNumber = currentRestructNum;
- }
-
public String getLevelAnddataType() {
return levelAnddataType;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
index bf1ecb3..f28950b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
@@ -97,8 +97,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
private Map<String, GenericDataType> complexTypes;
- private int currentRestructNumber;
-
/**
* It is column groups in below format
* 0,1~2~3,4,5,6~7~8,9
@@ -155,7 +153,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
databaseName = "";
columnGroupsString = "";
noDictionaryDims = "";
- currentRestructNumber = -1;
measureDataType = "";
taskNo = "";
factTimeStamp = "";
@@ -178,8 +175,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
retval.append(" ").append(XMLHandler.addTagValue("dimensionCount", dimensionCount));
retval.append(" ").append(XMLHandler.addTagValue("complexDimsCount", complexDimsCount));
retval.append(" ").append(XMLHandler.addTagValue("complexTypeString", complexTypeString));
- retval.append(" ")
- .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
retval.append(" ").append(XMLHandler.addTagValue("measureDataType", measureDataType));
retval.append(" ").append(XMLHandler.addTagValue("taskNo", taskNo));
retval.append(" ").append(XMLHandler.addTagValue("factTimeStamp", factTimeStamp));
@@ -207,8 +202,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
dimensionCount = XMLHandler.getTagValue(stepnode, "dimensionCount");
complexDimsCount = XMLHandler.getTagValue(stepnode, "complexDimsCount");
complexTypeString = XMLHandler.getTagValue(stepnode, "complexTypeString");
- currentRestructNumber =
- Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
measureDataType = XMLHandler.getTagValue(stepnode, "measureDataType");
taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
factTimeStamp = XMLHandler.getTagValue(stepnode, "factTimeStamp");
@@ -235,8 +228,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
rep.saveStepAttribute(idTransformation, idStep, "dimensionCount", dimensionCount);
rep.saveStepAttribute(idTransformation, idStep, "complexDimsCount", complexDimsCount);
rep.saveStepAttribute(idTransformation, idStep, "complexTypeString", complexTypeString);
- rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
- currentRestructNumber);
rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType);
rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
rep.saveStepAttribute(idTransformation, idStep, "factTimeStamp", factTimeStamp);
@@ -267,7 +258,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
dimensionCount = rep.getStepAttributeString(idStep, "dimensionCount");
complexDimsCount = rep.getStepAttributeString(idStep, "complexDimsCount");
complexTypeString = rep.getStepAttributeString(idStep, "complexTypeString");
- currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
measureDataType = rep.getStepAttributeString(idStep, "measureDataType");
taskNo = rep.getStepAttributeString(idStep, "taskNo");
factTimeStamp = rep.getStepAttributeString(idStep, "factTimeStamp");
@@ -400,20 +390,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
}
/**
- * @return the currentRestructNumber
- */
- public int getCurrentRestructNumber() {
- return currentRestructNumber;
- }
-
- /**
- * @param currentRestructNum the currentRestructNumber to set
- */
- public void setCurrentRestructNumber(int currentRestructNum) {
- this.currentRestructNumber = currentRestructNum;
- }
-
- /**
* @return
*/
public String getNoDictionaryDims() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
index c42ed0a..1d712ec 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
@@ -102,8 +102,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
*/
private String factDimLensString;
- private int currentRestructNumber = -1;
-
private String levelAnddataTypeString;
/**
* partitionID
@@ -139,7 +137,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
aggregatorClassString = "";
aggregatorString = "";
factDimLensString = "";
- currentRestructNumber = -1;
levelAnddataTypeString = "";
partitionID = "";
segmentId = "";
@@ -166,8 +163,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
retval.append(" ").append(XMLHandler.addTagValue("aggregatorString", aggregatorString));
retval.append(" ").append(XMLHandler.addTagValue("factDimLensString", factDimLensString));
retval.append(" ")
- .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
- retval.append(" ")
.append(XMLHandler.addTagValue("levelAnddataTypeString", levelAnddataTypeString));
retval.append(" ").append(XMLHandler.addTagValue("partitionID", partitionID));
retval.append(" ").append(XMLHandler.addTagValue("segmentId", segmentId));
@@ -197,8 +192,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
aggregatorString = XMLHandler.getTagValue(stepnode, "aggregatorString");
factDimLensString = XMLHandler.getTagValue(stepnode, "factDimLensString");
levelAnddataTypeString = XMLHandler.getTagValue(stepnode, "levelAnddataTypeString");
- currentRestructNumber =
- Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
segmentId = XMLHandler.getTagValue(stepnode, "segmentId");
taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
@@ -232,8 +225,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
rep.saveStepAttribute(idTransformation, idStep, "factDimLensString", factDimLensString);
rep.saveStepAttribute(idTransformation, idStep, "levelAnddataTypeString",
levelAnddataTypeString);
- rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
- currentRestructNumber);
rep.saveStepAttribute(idTransformation, idStep, "partitionID", partitionID);
rep.saveStepAttribute(idTransformation, idStep, "segmentId", segmentId);
rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
@@ -278,7 +269,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
aggregatorString = rep.getStepAttributeString(idStep, "aggregatorString");
factDimLensString = rep.getStepAttributeString(idStep, "factDimLensString");
levelAnddataTypeString = rep.getStepAttributeString(idStep, "levelAnddataTypeString");
- currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
partitionID = rep.getStepAttributeString(idStep, "partitionID");
segmentId = rep.getStepAttributeString(idStep, "segmentId");
taskNo = rep.getStepAttributeString(idStep, "taskNo");
@@ -518,20 +508,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
this.factDimLensString = factDimLensString1;
}
- /**
- * @return the currentRestructNumber
- */
- public int getCurrentRestructNumber() {
- return currentRestructNumber;
- }
-
- /**
- * @param currentRestructNum the currentRestructNumber to set
- */
- public void setCurrentRestructNumber(int currentRestructNum) {
- this.currentRestructNumber = currentRestructNum;
- }
-
public String getLevelAnddataTypeString() {
return levelAnddataTypeString;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
index a53fae4..9416134 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
@@ -97,8 +97,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
*/
private String updateMemberRequest;
- private int currentRestructNumber;
-
private String measureDataType;
private String noDictionaryDims;
@@ -134,7 +132,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
complexDimensionCount = "";
measureCount = "";
updateMemberRequest = "";
- currentRestructNumber = -1;
measureDataType = "";
partitionID = "";
segmentId = "";
@@ -163,8 +160,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
retval.append(" ").append(XMLHandler.addTagValue("measureCount", this.measureCount));
retval.append(" ")
.append(XMLHandler.addTagValue("isUpdateMemberRequest", this.updateMemberRequest));
- retval.append(" ")
- .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
retval.append(" ").append(XMLHandler.addTagValue("measureDataType", measureDataType));
retval.append(" ").append(XMLHandler.addTagValue("partitionID", partitionID));
retval.append(" ").append(XMLHandler.addTagValue("segmentId", segmentId));
@@ -195,8 +190,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
this.measureCount = XMLHandler.getTagValue(stepnode, "measureCount");
this.updateMemberRequest = XMLHandler.getTagValue(stepnode, "isUpdateMemberRequest");
this.measureDataType = XMLHandler.getTagValue(stepnode, "measureDataType");
- currentRestructNumber =
- Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
this.partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
this.segmentId = XMLHandler.getTagValue(stepnode, "segmentId");
this.taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
@@ -231,8 +224,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
rep.saveStepAttribute(idTransformation, idStep, "measureCount", this.measureCount);
rep.saveStepAttribute(idTransformation, idStep, "isUpdateMemberRequest",
this.updateMemberRequest);
- rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
- currentRestructNumber);
rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType);
rep.saveStepAttribute(idTransformation, idStep, "partitionID", partitionID);
rep.saveStepAttribute(idTransformation, idStep, "segmentId", segmentId);
@@ -267,8 +258,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
this.measureCount = rep.getStepAttributeString(idStep, "measureCount");
this.updateMemberRequest = rep.getStepAttributeString(idStep, "isUpdateMemberRequest");
this.measureDataType = rep.getStepAttributeString(idStep, "measureDataType");
- this.currentRestructNumber =
- (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
this.partitionID = rep.getStepAttributeString(idStep, "partitionID");
this.segmentId = rep.getStepAttributeString(idStep, "segmentId");
this.taskNo = rep.getStepAttributeString(idStep, "taskNo");
@@ -450,20 +439,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
this.updateMemberRequest = isUpdateMemberRequest;
}
- /**
- * @return the currentRestructNumber
- */
- public int getCurrentRestructNumber() {
- return currentRestructNumber;
- }
-
- /**
- * @param currentRestructNum the currentRestructNumber to set
- */
- public void setCurrentRestructNumber(int currentRestructNum) {
- this.currentRestructNumber = currentRestructNum;
- }
-
public String getMeasureDataType() {
return measureDataType;
}