You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/03/08 16:55:02 UTC
[01/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Repository: carbondata
Updated Branches:
refs/heads/master 3fb406618 -> 39fa1eb58
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index f51ced3..6a401d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -34,20 +34,16 @@ import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
/**
* Executor class for executing the query on the selected segments to be merged.
@@ -70,6 +66,9 @@ public class CarbonCompactionExecutor {
*/
private boolean restructuredBlockExists;
+ // converter for UTF8String and decimal conversion
+ private DataTypeConverter dataTypeConverter;
+
/**
* Constructor
*
@@ -82,13 +81,14 @@ public class CarbonCompactionExecutor {
public CarbonCompactionExecutor(Map<String, TaskBlockInfo> segmentMapping,
SegmentProperties segmentProperties, CarbonTable carbonTable,
Map<String, List<DataFileFooter>> dataFileMetadataSegMapping,
- boolean restructuredBlockExists) {
+ boolean restructuredBlockExists, DataTypeConverter dataTypeConverter) {
this.segmentMapping = segmentMapping;
this.destinationSegProperties = segmentProperties;
this.carbonTable = carbonTable;
this.dataFileMetadataSegMapping = dataFileMetadataSegMapping;
this.restructuredBlockExists = restructuredBlockExists;
- queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.dataTypeConverter = dataTypeConverter;
}
/**
@@ -100,7 +100,9 @@ public class CarbonCompactionExecutor {
List<RawResultIterator> resultList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<TableBlockInfo> list = null;
- queryModel = prepareQueryModel(list);
+ queryModel = carbonTable.createQueryModelWithProjectAllColumns(dataTypeConverter);
+ queryModel.setReadPageByPage(enablePageLevelReaderForCompaction());
+ queryModel.setForcedDetailRawQuery(true);
// iterate each seg ID
for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
String segmentId = taskMap.getKey();
@@ -156,7 +158,7 @@ public class CarbonCompactionExecutor {
* @param blockList
* @return
*/
- private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
+ private CarbonIterator<RowBatch> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
QueryExecutor queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
@@ -195,48 +197,6 @@ public class CarbonCompactionExecutor {
}
/**
- * Preparing of the query model.
- *
- * @param blockList
- * @return
- */
- private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
- QueryModel model = new QueryModel();
- model.setTableBlockInfos(blockList);
- model.setForcedDetailRawQuery(true);
- model.setFilterExpressionResolverTree(null);
- model.setConverter(DataTypeUtil.getDataTypeConverter());
- model.setReadPageByPage(enablePageLevelReaderForCompaction());
-
- List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getTableName());
- for (CarbonDimension dim : dimensions) {
- // check if dimension is deleted
- QueryDimension queryDimension = new QueryDimension(dim.getColName());
- queryDimension.setDimension(dim);
- dims.add(queryDimension);
- }
- model.setQueryDimension(dims);
-
- List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getTableName());
- for (CarbonMeasure carbonMeasure : measures) {
- // check if measure is deleted
- QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
- queryMeasure.setMeasure(carbonMeasure);
- msrs.add(queryMeasure);
- }
- model.setQueryMeasures(msrs);
- model.setQueryId(System.nanoTime() + "");
- model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- model.setTable(carbonTable);
- return model;
- }
-
- /**
* Whether to enable page level reader for compaction or not.
*/
private boolean enablePageLevelReaderForCompaction() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
index 79e9e5a..b6f12a5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/impl/QueryPartitionHelper.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.processing.partition.DataPartitioner;
import org.apache.carbondata.processing.partition.Partition;
@@ -46,9 +45,8 @@ public final class QueryPartitionHelper {
/**
* Get partitions applicable for query based on filters applied in query
*/
- public List<Partition> getPartitionsForQuery(CarbonQueryPlan queryPlan) {
- String tableUniqueName =
- CarbonTable.buildUniqueName(queryPlan.getDatabaseName(), queryPlan.getTableName());
+ public List<Partition> getPartitionsForQuery(String databaseName, String tableName) {
+ String tableUniqueName = CarbonTable.buildUniqueName(databaseName, tableName);
DataPartitioner dataPartitioner = partitionerMap.get(tableUniqueName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
index 36e022b..01db4f6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/AbstractCarbonQueryExecutor.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.processing.partition.spliter;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -26,19 +25,14 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.util.CarbonUtil;
public abstract class AbstractCarbonQueryExecutor {
@@ -47,8 +41,8 @@ public abstract class AbstractCarbonQueryExecutor {
LogServiceFactory.getLogService(AbstractCarbonQueryExecutor.class.getName());
protected CarbonTable carbonTable;
protected QueryModel queryModel;
- protected QueryExecutor queryExecutor;
- protected Map<String, TaskBlockInfo> segmentMapping;
+ private QueryExecutor queryExecutor;
+ Map<String, TaskBlockInfo> segmentMapping;
/**
* get executor and execute the query model.
@@ -56,7 +50,7 @@ public abstract class AbstractCarbonQueryExecutor {
* @param blockList
* @return
*/
- protected CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
+ CarbonIterator<RowBatch> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
@@ -64,46 +58,6 @@ public abstract class AbstractCarbonQueryExecutor {
}
/**
- * Preparing of the query model.
- *
- * @param blockList
- * @return
- */
- protected QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
- QueryModel model = new QueryModel();
- model.setTableBlockInfos(blockList);
- model.setForcedDetailRawQuery(true);
- model.setFilterExpressionResolverTree(null);
-
- List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- List<CarbonDimension> dimensions =
- carbonTable.getDimensionByTableName(carbonTable.getTableName());
- for (CarbonDimension dim : dimensions) {
- // check if dimension is deleted
- QueryDimension queryDimension = new QueryDimension(dim.getColName());
- queryDimension.setDimension(dim);
- dims.add(queryDimension);
- }
- model.setQueryDimension(dims);
-
- List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<CarbonMeasure> measures =
- carbonTable.getMeasureByTableName(carbonTable.getTableName());
- for (CarbonMeasure carbonMeasure : measures) {
- // check if measure is deleted
- QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
- queryMeasure.setMeasure(carbonMeasure);
- msrs.add(queryMeasure);
- }
- model.setQueryMeasures(msrs);
- model.setQueryId(System.nanoTime() + "");
- model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- model.setTable(carbonTable);
- return model;
- }
-
- /**
* Below method will be used
* for cleanup
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
index 6afec0b..b18207d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/CarbonSplitExecutor.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.result.iterator.PartitionSpliterRawResultIterator;
+import org.apache.carbondata.core.util.DataTypeConverterImpl;
/**
* Used to read carbon blocks when add/split partition
@@ -48,7 +49,8 @@ public class CarbonSplitExecutor extends AbstractCarbonQueryExecutor {
public List<PartitionSpliterRawResultIterator> processDataBlocks(String segmentId)
throws QueryExecutionException, IOException {
List<TableBlockInfo> list = null;
- queryModel = prepareQueryModel(list);
+ queryModel = carbonTable.createQueryModelWithProjectAllColumns(new DataTypeConverterImpl());
+ queryModel.setForcedDetailRawQuery(true);
List<PartitionSpliterRawResultIterator> resultList
= new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
TaskBlockInfo taskBlockInfo = segmentMapping.get(segmentId);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
index ec91472..4abdf3c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonQueryUtil.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.processing.partition.Partition;
import org.apache.carbondata.processing.partition.impl.DefaultLoadBalancer;
import org.apache.carbondata.processing.partition.impl.PartitionMultiFileImpl;
@@ -46,7 +46,7 @@ public class CarbonQueryUtil {
* It creates the one split for each region server.
*/
public static synchronized TableSplit[] getTableSplits(String databaseName, String tableName,
- CarbonQueryPlan queryPlan) {
+ QueryProjection queryPlan) {
//Just create splits depends on locations of region servers
List<Partition> allPartitions = null;
@@ -55,7 +55,7 @@ public class CarbonQueryUtil {
QueryPartitionHelper.getInstance().getAllPartitions(databaseName, tableName);
} else {
allPartitions =
- QueryPartitionHelper.getInstance().getPartitionsForQuery(queryPlan);
+ QueryPartitionHelper.getInstance().getPartitionsForQuery(databaseName, tableName);
}
TableSplit[] splits = new TableSplit[allPartitions.size()];
for (int i = 0; i < splits.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 5e0fce5..578f0cd 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -157,7 +157,7 @@ class StreamHandoffRDD[K, V](
CarbonTableInputFormat.setTableInfo(hadoopConf, carbonTable.getTableInfo)
val attemptContext = new TaskAttemptContextImpl(hadoopConf, attemptId)
val format = new CarbonTableInputFormat[Array[Object]]()
- val model = format.getQueryModel(inputSplit, attemptContext)
+ val model = format.createQueryModel(inputSplit, attemptContext)
val inputFormat = new CarbonStreamInputFormat
val streamReader = inputFormat.createRecordReader(inputSplit, attemptContext)
.asInstanceOf[CarbonStreamRecordReader]
[09/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 69f5ceb..22d1df1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -43,10 +43,9 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -64,8 +63,8 @@ import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -121,7 +120,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryProperties.queryStatisticsRecorder =
CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
- QueryUtil.resolveQueryModel(queryModel);
QueryStatistic queryStatistic = new QueryStatistic();
// sort the block info
// so block will be loaded in sorted order this will be required for
@@ -168,12 +166,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
// calculating the total number of aggeragted columns
- int measureCount = queryModel.getQueryMeasures().size();
+ int measureCount = queryModel.getProjectionMeasures().size();
int currentIndex = 0;
DataType[] dataTypes = new DataType[measureCount];
- for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+ for (ProjectionMeasure carbonMeasure : queryModel.getProjectionMeasures()) {
// adding the data type and aggregation type of all the measure this
// can be used
// to select the aggregator
@@ -198,9 +196,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
queryStatistic = new QueryStatistic();
// dictionary column unique column id to dictionary mapping
// which will be used to get column actual data
- queryProperties.columnToDictionayMapping = QueryUtil
- .getDimensionDictionaryDetail(queryModel.getQueryDimension(),
- queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier(),
+ queryProperties.columnToDictionayMapping =
+ QueryUtil.getDimensionDictionaryDetail(
+ queryModel.getProjectionDimensions(),
+ queryProperties.complexFilterDimension,
+ queryModel.getAbsoluteTableIdentifier(),
tableProvider);
queryStatistic
.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
@@ -263,8 +263,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// and query will be executed based on that infos
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
- BlockletDataRefNodeWrapper dataRefNode =
- (BlockletDataRefNodeWrapper) abstractIndex.getDataRefNode();
+ BlockletDataRefNode dataRefNode =
+ (BlockletDataRefNode) abstractIndex.getDataRefNode();
blockExecutionInfoList.add(getBlockExecutionInfoForBlock(queryModel, abstractIndex,
dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(),
dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(),
@@ -289,47 +289,45 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
* @return block execution info
* @throws QueryExecutionException any failure during block info creation
*/
- protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
+ private BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath,
String[] deleteDeltaFiles, String segmentId)
throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
- KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
// below is to get only those dimension in query which is present in the
// table block
- List<QueryDimension> currentBlockQueryDimensions = RestructureUtil
+ List<ProjectionDimension> projectDimensions = RestructureUtil
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo,
- queryModel.getQueryDimension(), tableBlockDimensions,
- segmentProperties.getComplexDimensions(), queryModel.getQueryMeasures().size());
+ queryModel.getProjectionDimensions(), tableBlockDimensions,
+ segmentProperties.getComplexDimensions(), queryModel.getProjectionMeasures().size());
blockExecutionInfo.setBlockId(
CarbonUtil.getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, segmentId));
blockExecutionInfo.setDeleteDeltaFilePath(deleteDeltaFiles);
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
- blockExecutionInfo.setQueryDimensions(currentBlockQueryDimensions
- .toArray(new QueryDimension[currentBlockQueryDimensions.size()]));
+ blockExecutionInfo.setProjectionDimensions(projectDimensions
+ .toArray(new ProjectionDimension[projectDimensions.size()]));
// get measures present in the current block
- List<QueryMeasure> currentBlockQueryMeasures =
+ List<ProjectionMeasure> currentBlockQueryMeasures =
getCurrentBlockQueryMeasures(blockExecutionInfo, queryModel, blockIndex);
- blockExecutionInfo.setQueryMeasures(
- currentBlockQueryMeasures.toArray(new QueryMeasure[currentBlockQueryMeasures.size()]));
+ blockExecutionInfo.setProjectionMeasures(
+ currentBlockQueryMeasures.toArray(new ProjectionMeasure[currentBlockQueryMeasures.size()]));
blockExecutionInfo.setDataBlock(blockIndex);
- blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// setting whether raw record query or not
blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
// total number dimension
blockExecutionInfo
- .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ .setTotalNumberDimensionToRead(
+ segmentProperties.getDimensionOrdinalToChunkMapping().size());
blockExecutionInfo.setPrefetchBlocklet(!queryModel.isReadPageByPage());
blockExecutionInfo
- .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
- blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
+ .setTotalNumberOfMeasureToRead(segmentProperties.getMeasuresOrdinalToChunkMapping().size());
blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
- .getComplexDimensionsMap(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(),
+ .getComplexDimensionsMap(projectDimensions,
+ segmentProperties.getDimensionOrdinalToChunkMapping(),
segmentProperties.getEachComplexDimColumnValueSize(),
queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
IndexKey startIndexKey = null;
@@ -363,45 +361,46 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// create a list of filter dimensions present in the current block
Set<CarbonDimension> currentBlockFilterDimensions =
getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
- int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
- currentBlockFilterDimensions, allProjectionListDimensionIdexes);
+ int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(
+ projectDimensions, segmentProperties.getDimensionOrdinalToChunkMapping(),
+ expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
- if (dimensionsBlockIndexes.length > 0) {
- numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
+ if (dimensionChunkIndexes.length > 0) {
+ numberOfElementToConsider = dimensionChunkIndexes[dimensionChunkIndexes.length - 1]
== segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?
- dimensionsBlockIndexes.length - 1 :
- dimensionsBlockIndexes.length;
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil
- .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider,
+ dimensionChunkIndexes.length - 1 :
+ dimensionChunkIndexes.length;
+ blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(
+ CarbonUtil.getRangeIndex(dimensionChunkIndexes, numberOfElementToConsider,
numberOfColumnToBeReadInOneIO));
} else {
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
+ blockExecutionInfo.setAllSelectedDimensionColumnIndexRange(new int[0][0]);
}
// get the list of updated filter measures present in the current block
- Set<CarbonMeasure> currentBlockFilterMeasures =
+ Set<CarbonMeasure> filterMeasures =
getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
// list of measures to be projected
List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
- int[] measureBlockIndexes = QueryUtil
- .getMeasureBlockIndexes(currentBlockQueryMeasures, expressionMeasures,
- segmentProperties.getMeasuresOrdinalToBlockMapping(), currentBlockFilterMeasures,
- allProjectionListMeasureIndexes);
- if (measureBlockIndexes.length > 0) {
+ int[] measureChunkIndexes = QueryUtil.getMeasureChunkIndexes(
+ currentBlockQueryMeasures, expressionMeasures,
+ segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures,
+ allProjectionListMeasureIndexes);
+ if (measureChunkIndexes.length > 0) {
- numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
+ numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1]
== segmentProperties.getMeasures().size() - 1 ?
- measureBlockIndexes.length - 1 :
- measureBlockIndexes.length;
+ measureChunkIndexes.length - 1 :
+ measureChunkIndexes.length;
// setting all the measure chunk indexes to be read from file
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil
- .getRangeIndex(measureBlockIndexes, numberOfElementToConsider,
+ blockExecutionInfo.setAllSelectedMeasureIndexRange(
+ CarbonUtil.getRangeIndex(
+ measureChunkIndexes, numberOfElementToConsider,
numberOfColumnToBeReadInOneIO));
} else {
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
+ blockExecutionInfo.setAllSelectedMeasureIndexRange(new int[0][0]);
}
// setting the indexes of list of dimension in projection list
blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(
@@ -413,27 +412,27 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
// setting the size of fixed key column (dictionary column)
blockExecutionInfo
- .setFixedLengthKeySize(getKeySize(currentBlockQueryDimensions, segmentProperties));
- Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
- List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
+ .setFixedLengthKeySize(getKeySize(projectDimensions, segmentProperties));
+ Set<Integer> dictionaryColumnChunkIndex = new HashSet<Integer>();
+ List<Integer> noDictionaryColumnChunkIndex = new ArrayList<Integer>();
// get the block index to be read from file for query dimension
// for both dictionary columns and no dictionary columns
- QueryUtil.fillQueryDimensionsBlockIndexes(currentBlockQueryDimensions,
- segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex,
- noDictionaryColumnBlockIndex);
- int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(
- dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
+ QueryUtil.fillQueryDimensionChunkIndexes(projectDimensions,
+ segmentProperties.getDimensionOrdinalToChunkMapping(), dictionaryColumnChunkIndex,
+ noDictionaryColumnChunkIndex);
+ int[] queryDictionaryColumnChunkIndexes = ArrayUtils.toPrimitive(
+ dictionaryColumnChunkIndex.toArray(new Integer[dictionaryColumnChunkIndex.size()]));
// need to sort the dictionary column as for all dimension
// column key will be filled based on key order
- Arrays.sort(queryDictionaryColumnBlockIndexes);
- blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
+ Arrays.sort(queryDictionaryColumnChunkIndexes);
+ blockExecutionInfo.setDictionaryColumnChunkIndex(queryDictionaryColumnChunkIndexes);
// setting the no dictionary column block indexes
- blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(
- noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
+ blockExecutionInfo.setNoDictionaryColumnChunkIndexes(ArrayUtils.toPrimitive(
+ noDictionaryColumnChunkIndex.toArray(new Integer[noDictionaryColumnChunkIndex.size()])));
// setting each column value size
blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
blockExecutionInfo.setComplexColumnParentBlockIndexes(
- getComplexDimensionParentBlockIndexes(currentBlockQueryDimensions));
+ getComplexDimensionParentBlockIndexes(projectDimensions));
blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
try {
// to set column group and its key structure info which will be used
@@ -441,15 +440,15 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// for getting the column group column data in case of final row
// and in case of dimension aggregation
blockExecutionInfo.setColumnGroupToKeyStructureInfo(
- QueryUtil.getColumnGroupKeyStructureInfo(currentBlockQueryDimensions, segmentProperties));
+ QueryUtil.getColumnGroupKeyStructureInfo(projectDimensions, segmentProperties));
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
// set actual query dimensions and measures. It may differ in case of restructure scenarios
- blockExecutionInfo.setActualQueryDimensions(queryModel.getQueryDimension()
- .toArray(new QueryDimension[queryModel.getQueryDimension().size()]));
- blockExecutionInfo.setActualQueryMeasures(queryModel.getQueryMeasures()
- .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
+ blockExecutionInfo.setActualQueryDimensions(queryModel.getProjectionDimensions()
+ .toArray(new ProjectionDimension[queryModel.getProjectionDimensions().size()]));
+ blockExecutionInfo.setActualQueryMeasures(queryModel.getProjectionMeasures()
+ .toArray(new ProjectionMeasure[queryModel.getProjectionMeasures().size()]));
DataTypeUtil.setDataTypeConverter(queryModel.getConverter());
return blockExecutionInfo;
}
@@ -464,7 +463,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
* @param blockMetadataInfo block metadata info
* @return key size
*/
- private int getKeySize(List<QueryDimension> queryDimension,
+ private int getKeySize(List<ProjectionDimension> queryDimension,
SegmentProperties blockMetadataInfo) {
// add the dimension block ordinal for each dictionary column
// existing in the current block dimensions. Set is used because in case of column groups
@@ -475,12 +474,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
while (counter < queryDimension.size()) {
if (queryDimension.get(counter).getDimension().getNumberOfChild() > 0) {
counter += queryDimension.get(counter).getDimension().getNumberOfChild();
- continue;
} else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
Encoding.DICTIONARY)) {
counter++;
} else {
- fixedLengthDimensionOrdinal.add(blockMetadataInfo.getDimensionOrdinalToBlockMapping()
+ fixedLengthDimensionOrdinal.add(blockMetadataInfo.getDimensionOrdinalToChunkMapping()
.get(queryDimension.get(counter).getDimension().getOrdinal()));
counter++;
}
@@ -502,26 +500,26 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
/**
* Below method will be used to get the measures present in the current block
*
- * @param blockExecutionInfo
+ * @param executionInfo
* @param queryModel query model
* @param tableBlock table block
* @return
*/
- private List<QueryMeasure> getCurrentBlockQueryMeasures(BlockExecutionInfo blockExecutionInfo,
+ private List<ProjectionMeasure> getCurrentBlockQueryMeasures(BlockExecutionInfo executionInfo,
QueryModel queryModel, AbstractIndex tableBlock) throws QueryExecutionException {
// getting the measure info which will be used while filling up measure data
- List<QueryMeasure> updatedQueryMeasures = RestructureUtil
- .createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo,
- queryModel.getQueryMeasures(), tableBlock.getSegmentProperties().getMeasures());
+ List<ProjectionMeasure> updatedQueryMeasures = RestructureUtil
+ .createMeasureInfoAndGetCurrentBlockQueryMeasures(executionInfo,
+ queryModel.getProjectionMeasures(), tableBlock.getSegmentProperties().getMeasures());
// setting the measure aggregator for all aggregation function selected
// in query
- blockExecutionInfo.getMeasureInfo().setMeasureDataTypes(queryProperties.measureDataTypes);
+ executionInfo.getMeasureInfo().setMeasureDataTypes(queryProperties.measureDataTypes);
return updatedQueryMeasures;
}
- private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
+ private int[] getComplexDimensionParentBlockIndexes(List<ProjectionDimension> queryDimensions) {
List<Integer> parentBlockIndexList = new ArrayList<Integer>();
- for (QueryDimension queryDimension : queryDimensions) {
+ for (ProjectionDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().getDataType().isComplexType()) {
parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
index 931b1ab..93d696b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
@@ -23,7 +23,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator;
/**
@@ -31,10 +31,10 @@ import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator
* For executing the detail query it will pass all the block execution
* info to detail query result iterator and iterator will be returned
*/
-public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
+public class DetailQueryExecutor extends AbstractQueryExecutor<RowBatch> {
@Override
- public CarbonIterator<BatchResult> execute(QueryModel queryModel)
+ public CarbonIterator<RowBatch> execute(QueryModel queryModel)
throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
this.queryIterator = new DetailQueryResultIterator(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 65f2838..06adf8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -21,13 +21,11 @@ import java.util.Map;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
/**
* Below class will have all the properties which needed during query execution
@@ -41,14 +39,6 @@ public class BlockExecutionInfo {
private AbstractIndex blockIndex;
/**
- * each segment key size can be different and in that case we need to update
- * the fixed key with latest segment key generator. so this property will
- * tell whether this is required or not if key size is same then it is not
- * required
- */
- private boolean isFixedKeyUpdateRequired;
-
- /**
* below to store all the information required for measures during query
* execution
*/
@@ -73,22 +63,22 @@ public class BlockExecutionInfo {
/**
* total number of dimension in block
*/
- private int totalNumberDimensionBlock;
+ private int totalNumberDimensionToRead;
/**
* total number of measure in block
*/
- private int totalNumberOfMeasureBlock;
+ private int totalNumberOfMeasureToRead;
/**
* will be used to read the dimension block from file
*/
- private int[][] allSelectedDimensionBlocksIndexes;
+ private int[][] allSelectedDimensionColumnIndexRange;
/**
* will be used to read the measure block from file
*/
- private int[][] allSelectedMeasureBlocksIndexes;
+ private int[][] allSelectedMeasureIndexRange;
/**
* list of dimension present in the projection
@@ -118,16 +108,11 @@ public class BlockExecutionInfo {
/**
* dictionary column block indexes based on query
*/
- private int[] dictionaryColumnBlockIndex;
+ private int[] dictionaryColumnChunkIndex;
/**
* no dictionary column block indexes in based on the query order
*/
- private int[] noDictionaryBlockIndexes;
-
- /**
- * key generator used for generating the table block fixed length key
- */
- private KeyGenerator blockKeyGenerator;
+ private int[] noDictionaryColumnChunkIndexes;
/**
* each column value size
@@ -180,23 +165,23 @@ public class BlockExecutionInfo {
* list of dimension present in the current block. This will be
* different in case of restructured block
*/
- private QueryDimension[] queryDimensions;
+ private ProjectionDimension[] projectionDimensions;
/**
* list of dimension selected for in query
*/
- private QueryDimension[] actualQueryDimensions;
+ private ProjectionDimension[] actualQueryDimensions;
/**
* list of dimension present in the current block. This will be
* different in case of restructured block
*/
- private QueryMeasure[] queryMeasures;
+ private ProjectionMeasure[] projectionMeasures;
/**
* list of measure selected in query
*/
- private QueryMeasure[] actualQueryMeasures;
+ private ProjectionMeasure[] actualQueryMeasures;
/**
* variable to maintain dimension existence and default value info
@@ -214,11 +199,6 @@ public class BlockExecutionInfo {
private boolean isRestructuredBlock;
/**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
-
- /**
* delete delta file path
*/
private String[] deleteDeltaFilePath;
@@ -229,13 +209,6 @@ public class BlockExecutionInfo {
private boolean prefetchBlocklet = true;
private Map<String, DeleteDeltaVo> deletedRecordsMap;
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- }
/**
* @param blockIndex the tableBlock to set
@@ -245,20 +218,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the isFixedKeyUpdateRequired
- */
- public boolean isFixedKeyUpdateRequired() {
- return isFixedKeyUpdateRequired;
- }
-
- /**
- * @param isFixedKeyUpdateRequired the isFixedKeyUpdateRequired to set
- */
- public void setFixedKeyUpdateRequired(boolean isFixedKeyUpdateRequired) {
- this.isFixedKeyUpdateRequired = isFixedKeyUpdateRequired;
- }
-
- /**
* @return the aggregatorInfos
*/
public MeasureInfo getMeasureInfo() {
@@ -301,59 +260,60 @@ public class BlockExecutionInfo {
}
/**
- * @return the totalNumberDimensionBlock
+ * @return the totalNumberDimensionToRead
*/
- public int getTotalNumberDimensionBlock() {
- return totalNumberDimensionBlock;
+ public int getTotalNumberDimensionToRead() {
+ return totalNumberDimensionToRead;
}
/**
- * @param totalNumberDimensionBlock the totalNumberDimensionBlock to set
+ * @param totalNumberDimensionToRead the totalNumberDimensionToRead to set
*/
- public void setTotalNumberDimensionBlock(int totalNumberDimensionBlock) {
- this.totalNumberDimensionBlock = totalNumberDimensionBlock;
+ public void setTotalNumberDimensionToRead(int totalNumberDimensionToRead) {
+ this.totalNumberDimensionToRead = totalNumberDimensionToRead;
}
/**
- * @return the totalNumberOfMeasureBlock
+ * @return the totalNumberOfMeasureToRead
*/
- public int getTotalNumberOfMeasureBlock() {
- return totalNumberOfMeasureBlock;
+ public int getTotalNumberOfMeasureToRead() {
+ return totalNumberOfMeasureToRead;
}
/**
- * @param totalNumberOfMeasureBlock the totalNumberOfMeasureBlock to set
+ * @param totalNumberOfMeasureToRead the totalNumberOfMeasureToRead to set
*/
- public void setTotalNumberOfMeasureBlock(int totalNumberOfMeasureBlock) {
- this.totalNumberOfMeasureBlock = totalNumberOfMeasureBlock;
+ public void setTotalNumberOfMeasureToRead(int totalNumberOfMeasureToRead) {
+ this.totalNumberOfMeasureToRead = totalNumberOfMeasureToRead;
}
/**
- * @return the allSelectedDimensionBlocksIndexes
+ * @return the allSelectedDimensionColumnIndexRange
*/
- public int[][] getAllSelectedDimensionBlocksIndexes() {
- return allSelectedDimensionBlocksIndexes;
+ public int[][] getAllSelectedDimensionColumnIndexRange() {
+ return allSelectedDimensionColumnIndexRange;
}
/**
- * @param allSelectedDimensionBlocksIndexes the allSelectedDimensionBlocksIndexes to set
+ * @param allSelectedDimensionColumnIndexRange the allSelectedDimensionColumnIndexRange to set
*/
- public void setAllSelectedDimensionBlocksIndexes(int[][] allSelectedDimensionBlocksIndexes) {
- this.allSelectedDimensionBlocksIndexes = allSelectedDimensionBlocksIndexes;
+ public void setAllSelectedDimensionColumnIndexRange(int[][] allSelectedDimensionColumnIndexRange)
+ {
+ this.allSelectedDimensionColumnIndexRange = allSelectedDimensionColumnIndexRange;
}
/**
- * @return the allSelectedMeasureBlocksIndexes
+ * @return the allSelectedMeasureIndexRange
*/
- public int[][] getAllSelectedMeasureBlocksIndexes() {
- return allSelectedMeasureBlocksIndexes;
+ public int[][] getAllSelectedMeasureIndexRange() {
+ return allSelectedMeasureIndexRange;
}
/**
- * @param allSelectedMeasureBlocksIndexes the allSelectedMeasureBlocksIndexes to set
+ * @param allSelectedMeasureIndexRange the allSelectedMeasureIndexRange to set
*/
- public void setAllSelectedMeasureBlocksIndexes(int[][] allSelectedMeasureBlocksIndexes) {
- this.allSelectedMeasureBlocksIndexes = allSelectedMeasureBlocksIndexes;
+ public void setAllSelectedMeasureIndexRange(int[][] allSelectedMeasureIndexRange) {
+ this.allSelectedMeasureIndexRange = allSelectedMeasureIndexRange;
}
/**
@@ -413,20 +373,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the tableBlockKeyGenerator
- */
- public KeyGenerator getBlockKeyGenerator() {
- return blockKeyGenerator;
- }
-
- /**
- * @param tableBlockKeyGenerator the tableBlockKeyGenerator to set
- */
- public void setBlockKeyGenerator(KeyGenerator tableBlockKeyGenerator) {
- this.blockKeyGenerator = tableBlockKeyGenerator;
- }
-
- /**
* @return the eachColumnValueSize
*/
public int[] getEachColumnValueSize() {
@@ -441,31 +387,31 @@ public class BlockExecutionInfo {
}
/**
- * @return the dictionaryColumnBlockIndex
+ * @return the dictionaryColumnChunkIndex
*/
- public int[] getDictionaryColumnBlockIndex() {
- return dictionaryColumnBlockIndex;
+ public int[] getDictionaryColumnChunkIndex() {
+ return dictionaryColumnChunkIndex;
}
/**
- * @param dictionaryColumnBlockIndex the dictionaryColumnBlockIndex to set
+ * @param dictionaryColumnChunkIndex the dictionaryColumnChunkIndex to set
*/
- public void setDictionaryColumnBlockIndex(int[] dictionaryColumnBlockIndex) {
- this.dictionaryColumnBlockIndex = dictionaryColumnBlockIndex;
+ public void setDictionaryColumnChunkIndex(int[] dictionaryColumnChunkIndex) {
+ this.dictionaryColumnChunkIndex = dictionaryColumnChunkIndex;
}
/**
- * @return the noDictionaryBlockIndexes
+ * @return the noDictionaryColumnChunkIndexes
*/
- public int[] getNoDictionaryBlockIndexes() {
- return noDictionaryBlockIndexes;
+ public int[] getNoDictionaryColumnChunkIndexes() {
+ return noDictionaryColumnChunkIndexes;
}
/**
- * @param noDictionaryBlockIndexes the noDictionaryBlockIndexes to set
+ * @param noDictionaryColumnChunkIndexes the noDictionaryColumnChunkIndexes to set
*/
- public void setNoDictionaryBlockIndexes(int[] noDictionaryBlockIndexes) {
- this.noDictionaryBlockIndexes = noDictionaryBlockIndexes;
+ public void setNoDictionaryColumnChunkIndexes(int[] noDictionaryColumnChunkIndexes) {
+ this.noDictionaryColumnChunkIndexes = noDictionaryColumnChunkIndexes;
}
/**
@@ -519,20 +465,20 @@ public class BlockExecutionInfo {
this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes;
}
- public QueryDimension[] getQueryDimensions() {
- return queryDimensions;
+ public ProjectionDimension[] getProjectionDimensions() {
+ return projectionDimensions;
}
- public void setQueryDimensions(QueryDimension[] queryDimensions) {
- this.queryDimensions = queryDimensions;
+ public void setProjectionDimensions(ProjectionDimension[] projectionDimensions) {
+ this.projectionDimensions = projectionDimensions;
}
- public QueryMeasure[] getQueryMeasures() {
- return queryMeasures;
+ public ProjectionMeasure[] getProjectionMeasures() {
+ return projectionMeasures;
}
- public void setQueryMeasures(QueryMeasure[] queryMeasures) {
- this.queryMeasures = queryMeasures;
+ public void setProjectionMeasures(ProjectionMeasure[] projectionMeasures) {
+ this.projectionMeasures = projectionMeasures;
}
/**
@@ -579,7 +525,8 @@ public class BlockExecutionInfo {
this.vectorBatchCollector = vectorBatchCollector;
}
- public String getBlockId() {
+ // Return file name and path, like Part0/Segment_0/part-0-0_batchno0-0-1517155583332.carbondata
+ public String getBlockIdString() {
return blockId;
}
@@ -603,19 +550,19 @@ public class BlockExecutionInfo {
this.dimensionInfo = dimensionInfo;
}
- public QueryDimension[] getActualQueryDimensions() {
+ public ProjectionDimension[] getActualQueryDimensions() {
return actualQueryDimensions;
}
- public void setActualQueryDimensions(QueryDimension[] actualQueryDimensions) {
+ public void setActualQueryDimensions(ProjectionDimension[] actualQueryDimensions) {
this.actualQueryDimensions = actualQueryDimensions;
}
- public QueryMeasure[] getActualQueryMeasures() {
+ public ProjectionMeasure[] getActualQueryMeasures() {
return actualQueryMeasures;
}
- public void setActualQueryMeasures(QueryMeasure[] actualQueryMeasures) {
+ public void setActualQueryMeasures(ProjectionMeasure[] actualQueryMeasures) {
this.actualQueryMeasures = actualQueryMeasures;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 8eb0a0d..00cdfe7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -60,9 +60,8 @@ import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -83,7 +82,7 @@ public class QueryUtil {
* @param keyGenerator key generator
* @return masked key
*/
- public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions,
+ public static int[] getMaskedByteRange(List<ProjectionDimension> queryDimensions,
KeyGenerator keyGenerator) {
Set<Integer> byteRangeSet = new TreeSet<Integer>();
int[] byteRange = null;
@@ -167,7 +166,7 @@ public class QueryUtil {
* @return max key for dimension
* @throws KeyGenException if any problem while generating the key
*/
- public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions,
+ public static byte[] getMaxKeyBasedOnDimensions(List<ProjectionDimension> queryDimensions,
KeyGenerator generator) throws KeyGenException {
long[] max = new long[generator.getDimCount()];
Arrays.fill(max, 0L);
@@ -209,46 +208,46 @@ public class QueryUtil {
* on query dimension
*
* @param queryDimensions query dimension
- * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension
+ * @param dimensionOrdinalToChunkMapping mapping of dimension block in file to query dimension
* @return block index of file
*/
- public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+ public static int[] getDimensionChunkIndexes(List<ProjectionDimension> queryDimensions,
+ Map<Integer, Integer> dimensionOrdinalToChunkMapping,
List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
Set<Integer> allProjectionListDimensionIndexes) {
// using set as in row group columns will point to same block
- Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
+ Set<Integer> dimensionChunkIndex = new HashSet<Integer>();
Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
- int blockIndex = 0;
+ int chunkIndex = 0;
for (int i = 0; i < queryDimensions.size(); i++) {
if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
continue;
}
Integer dimensionOrdinal = queryDimensions.get(i).getDimension().getOrdinal();
- allProjectionListDimensionIndexes.add(dimensionOrdinalToBlockMapping.get(dimensionOrdinal));
+ allProjectionListDimensionIndexes.add(dimensionOrdinalToChunkMapping.get(dimensionOrdinal));
if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
addChildrenBlockIndex(allProjectionListDimensionIndexes,
queryDimensions.get(i).getDimension());
}
if (!filterDimensionOrdinal.contains(dimensionOrdinal)) {
- blockIndex = dimensionOrdinalToBlockMapping.get(dimensionOrdinal);
- dimensionBlockIndex.add(blockIndex);
+ chunkIndex = dimensionOrdinalToChunkMapping.get(dimensionOrdinal);
+ dimensionChunkIndex.add(chunkIndex);
if (queryDimensions.get(i).getDimension().getNumberOfChild() > 0) {
- addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ addChildrenBlockIndex(dimensionChunkIndex, queryDimensions.get(i).getDimension());
}
}
}
for (int i = 0; i < customAggregationDimension.size(); i++) {
- blockIndex =
- dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+ chunkIndex =
+ dimensionOrdinalToChunkMapping.get(customAggregationDimension.get(i).getOrdinal());
// not adding the children dimension as dimension aggregation
// is not push down in case of complex dimension
- dimensionBlockIndex.add(blockIndex);
+ dimensionChunkIndex.add(chunkIndex);
}
int[] dimensionIndex = ArrayUtils
- .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
+ .toPrimitive(dimensionChunkIndex.toArray(new Integer[dimensionChunkIndex.size()]));
Arrays.sort(dimensionIndex);
return dimensionIndex;
}
@@ -278,7 +277,7 @@ public class QueryUtil {
* @throws IOException
*/
public static Map<String, Dictionary> getDimensionDictionaryDetail(
- List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
+ List<ProjectionDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
AbsoluteTableIdentifier absoluteTableIdentifier, TableProvider tableProvider)
throws IOException {
// to store complex dimension and its child id unique column id list, this is required as
@@ -433,23 +432,23 @@ public class QueryUtil {
* @param ordinalToBlockIndexMapping measure ordinal to block mapping
* @return block indexes
*/
- public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
+ public static int[] getMeasureChunkIndexes(List<ProjectionMeasure> queryMeasures,
List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
Set<CarbonMeasure> filterMeasures, List<Integer> allProjectionListMeasureIdexes) {
- Set<Integer> measureBlockIndex = new HashSet<Integer>();
+ Set<Integer> measureChunkIndex = new HashSet<Integer>();
Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
for (int i = 0; i < queryMeasures.size(); i++) {
Integer measureOrdinal = queryMeasures.get(i).getMeasure().getOrdinal();
allProjectionListMeasureIdexes.add(measureOrdinal);
if (!filterMeasureOrdinal.contains(measureOrdinal)) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
+ measureChunkIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
}
}
for (int i = 0; i < expressionMeasure.size(); i++) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+ measureChunkIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
}
int[] measureIndexes =
- ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
+ ArrayUtils.toPrimitive(measureChunkIndex.toArray(new Integer[measureChunkIndex.size()]));
Arrays.sort(measureIndexes);
return measureIndexes;
}
@@ -462,8 +461,8 @@ public class QueryUtil {
* @param queryDimensions query dimension
* @return sort dimension indexes
*/
- public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions,
- List<QueryDimension> queryDimensions) {
+ public static byte[] getSortDimensionIndexes(List<ProjectionDimension> sortedDimensions,
+ List<ProjectionDimension> queryDimensions) {
byte[] sortedDims = new byte[queryDimensions.size()];
int indexOf = 0;
for (int i = 0; i < sortedDims.length; i++) {
@@ -485,7 +484,7 @@ public class QueryUtil {
* @throws KeyGenException if problem while key generation
*/
public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
- List<QueryDimension> queryDimensions, SegmentProperties segmentProperties)
+ List<ProjectionDimension> queryDimensions, SegmentProperties segmentProperties)
throws KeyGenException {
Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
// get column group id and its ordinal mapping of column group
@@ -521,7 +520,7 @@ public class QueryUtil {
restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
.toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
rowGroupToItsRSInfo
- .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
+ .put(segmentProperties.getDimensionOrdinalToChunkMapping().get(ordinal.get(0)),
restructureInfos);
}
return rowGroupToItsRSInfo;
@@ -552,16 +551,16 @@ public class QueryUtil {
* stored in bit level
*/
private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
- List<QueryDimension> origdimensions) {
+ List<ProjectionDimension> origDimensions) {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size());
- dimensions.addAll(origdimensions);
+ List<ProjectionDimension> dimensions = new ArrayList<>(origDimensions.size());
+ dimensions.addAll(origDimensions);
/*
* sort based on column group id
*/
- Collections.sort(dimensions, new Comparator<QueryDimension>() {
+ Collections.sort(dimensions, new Comparator<ProjectionDimension>() {
- @Override public int compare(QueryDimension o1, QueryDimension o2) {
+ @Override public int compare(ProjectionDimension o1, ProjectionDimension o2) {
return Integer
.compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
}
@@ -636,70 +635,26 @@ public class QueryUtil {
* dictionary column. This is done for specific purpose so that in one
* iteration we will be able to fill both type dimension block indexes
*
- * @param queryDimensions dimension present in the query
- * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping
- * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes
- * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes
+ * @param projectDimensions dimension present in the query
+ * @param columnOrdinalToChunkIndexMapping column ordinal to block index mapping
+ * @param dictionaryDimensionChunkIndex list to store dictionary column block indexes
+ * @param noDictionaryDimensionChunkIndex list to store no dictionary block indexes
*/
- public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
- Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
- for (QueryDimension queryDimension : queryDimensions) {
+ public static void fillQueryDimensionChunkIndexes(
+ List<ProjectionDimension> projectDimensions,
+ Map<Integer, Integer> columnOrdinalToChunkIndexMapping,
+ Set<Integer> dictionaryDimensionChunkIndex,
+ List<Integer> noDictionaryDimensionChunkIndex) {
+ for (ProjectionDimension queryDimension : projectDimensions) {
if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
&& queryDimension.getDimension().getNumberOfChild() == 0) {
- dictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ dictionaryDimensionChunkIndex
+ .add(columnOrdinalToChunkIndexMapping.get(queryDimension.getDimension().getOrdinal()));
} else if (
!CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
&& queryDimension.getDimension().getNumberOfChild() == 0) {
- noDictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- }
- }
- }
-
- /**
- * Below method will be used to resolve the query model
- * resolve will be setting the actual dimension and measure object
- * as from driver only column name will be passes to avoid the heavy object
- * serialization
- *
- * @param queryModel query model
- */
- public static void resolveQueryModel(QueryModel queryModel) {
- CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
- // TODO need to load the table from table identifier
- CarbonTable carbonTable = queryModel.getTable();
- String tableName =
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
- // resolve query dimension
- for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
- queryDimension
- .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
- }
- // resolve query measure
- for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
- // in case of count start column name will be count * so
- // first need to check any measure is present or not and as if measure
- // if measure is present and if first measure is not a default
- // measure than add measure otherwise
- // than add first dimension as a measure
- //as currently if measure is not present then
- //we are adding default measure so first condition will
- //never come false but if in future we can remove so not removing first if check
- if (queryMeasure.getColumnName().equals("count(*)")) {
- if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable
- .getMeasureByTableName(tableName).get(0).getColName()
- .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
- queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
- } else {
- CarbonMeasure dummyMeasure = new CarbonMeasure(
- carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
- queryMeasure.setMeasure(dummyMeasure);
- }
- } else {
- queryMeasure
- .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
+ noDictionaryDimensionChunkIndex
+ .add(columnOrdinalToChunkIndexMapping.get(queryDimension.getDimension().getOrdinal()));
}
}
}
@@ -759,11 +714,11 @@ public class QueryUtil {
* @return complex dimension and query type
*/
public static Map<Integer, GenericQueryType> getComplexDimensionsMap(
- List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
+ List<ProjectionDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap,
Set<CarbonDimension> filterDimensions) {
Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>();
- for (QueryDimension dimension : queryDimensions) {
+ for (ProjectionDimension dimension : queryDimensions) {
CarbonDimension actualDimension = dimension.getDimension();
if (actualDimension.getNumberOfChild() == 0) {
continue;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index a029986..2712cbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -61,12 +61,12 @@ public class RestructureUtil {
* @param tableComplexDimension
* @return list of query dimension which is present in the table block
*/
- public static List<QueryDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
- BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions,
+ public static List<ProjectionDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(
+ BlockExecutionInfo blockExecutionInfo, List<ProjectionDimension> queryDimensions,
List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension,
int measureCount) {
- List<QueryDimension> presentDimension =
- new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ List<ProjectionDimension> presentDimension =
+ new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
boolean[] isDimensionExists = new boolean[queryDimensions.size()];
Object[] defaultValues = new Object[queryDimensions.size()];
// create dimension information instance
@@ -76,7 +76,7 @@ public class RestructureUtil {
int newNoDictionaryColumnCount = 0;
// selecting only those dimension which is present in the query
int dimIndex = 0;
- for (QueryDimension queryDimension : queryDimensions) {
+ for (ProjectionDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
presentDimension.add(queryDimension);
isDimensionExists[dimIndex] = true;
@@ -85,15 +85,14 @@ public class RestructureUtil {
} else {
for (CarbonDimension tableDimension : tableBlockDimensions) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
- QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
+ ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
tableDimension.getColumnSchema()
.setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
tableDimension.getColumnSchema()
.setScale(queryDimension.getDimension().getColumnSchema().getScale());
tableDimension.getColumnSchema()
.setDefaultValue(queryDimension.getDimension().getDefaultValue());
- currentBlockDimension.setDimension(tableDimension);
- currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
+ currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
dimensionInfo.dataType[currentBlockDimension.getQueryOrder()] =
@@ -108,11 +107,10 @@ public class RestructureUtil {
}
for (CarbonDimension tableDimension : tableComplexDimension) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
- QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
+ ProjectionDimension currentBlockDimension = new ProjectionDimension(tableDimension);
// TODO: for complex dimension set scale and precision by traversing
// the child dimensions
- currentBlockDimension.setDimension(tableDimension);
- currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
+ currentBlockDimension.setOrdinal(queryDimension.getOrdinal());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
dimensionInfo.dataType[currentBlockDimension.getQueryOrder()] =
@@ -250,39 +248,6 @@ public class RestructureUtil {
}
/**
- * Below method is to add dimension children for complex type dimension as
- * internally we are creating dimension column for each each complex
- * dimension so when complex query dimension request will come in the query,
- * we need to add its children as it is hidden from the user For example if
- * complex dimension is of Array of String[2] so we are storing 3 dimension
- * and when user will query for complex type i.e. array type we need to add
- * its children and then we will read respective block and create a tuple
- * based on all three dimension
- *
- * @param queryDimensions current query dimensions
- * @param tableBlockDimensions dimensions which is present in the table block
- * @return updated dimension(after adding complex type children)
- */
- public static List<CarbonDimension> addChildrenForComplexTypeDimension(
- List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) {
- List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>();
- int numberOfChildren = 0;
- for (CarbonDimension queryDimension : queryDimensions) {
- // if number of child is zero, then it is not a complex dimension
- // so directly add it query dimension
- if (queryDimension.getNumberOfChild() == 0) {
- updatedQueryDimension.add(queryDimension);
- }
- // if number of child is more than 1 then add all its children
- numberOfChildren = queryDimension.getOrdinal() + queryDimension.getNumberOfChild();
- for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) {
- updatedQueryDimension.add(tableBlockDimensions.get(j));
- }
- }
- return updatedQueryDimension;
- }
-
- /**
* Method for computing measure default value based on the data type
*
* @param columnSchema
@@ -375,30 +340,29 @@ public class RestructureUtil {
* @param currentBlockMeasures current block measures
* @return measures present in the block
*/
- public static List<QueryMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
- BlockExecutionInfo blockExecutionInfo, List<QueryMeasure> queryMeasures,
+ public static List<ProjectionMeasure> createMeasureInfoAndGetCurrentBlockQueryMeasures(
+ BlockExecutionInfo blockExecutionInfo, List<ProjectionMeasure> queryMeasures,
List<CarbonMeasure> currentBlockMeasures) {
MeasureInfo measureInfo = new MeasureInfo();
- List<QueryMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
+ List<ProjectionMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
int numberOfMeasureInQuery = queryMeasures.size();
List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
Object[] defaultValues = new Object[numberOfMeasureInQuery];
boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
int index = 0;
- for (QueryMeasure queryMeasure : queryMeasures) {
+ for (ProjectionMeasure queryMeasure : queryMeasures) {
// if query measure exists in current dimension measures
// then setting measure exists is true
// otherwise adding a default value of a measure
for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) {
- QueryMeasure currentBlockMeasure = new QueryMeasure(carbonMeasure.getColName());
+ ProjectionMeasure currentBlockMeasure = new ProjectionMeasure(carbonMeasure);
carbonMeasure.getColumnSchema().setDataType(queryMeasure.getMeasure().getDataType());
carbonMeasure.getColumnSchema().setPrecision(queryMeasure.getMeasure().getPrecision());
carbonMeasure.getColumnSchema().setScale(queryMeasure.getMeasure().getScale());
carbonMeasure.getColumnSchema()
.setDefaultValue(queryMeasure.getMeasure().getDefaultValue());
- currentBlockMeasure.setMeasure(carbonMeasure);
- currentBlockMeasure.setQueryOrder(queryMeasure.getQueryOrder());
+ currentBlockMeasure.setOrdinal(queryMeasure.getOrdinal());
presentMeasure.add(currentBlockMeasure);
measureOrdinalList.add(carbonMeasure.getOrdinal());
measureExistsInCurrentBlock[index] = true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
index 981efb5..5cd2d34 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -70,10 +70,6 @@ public class ColumnExpression extends LeafExpression {
return columnName;
}
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
public boolean isDimension() {
return isDimension;
}
@@ -102,10 +98,6 @@ public class ColumnExpression extends LeafExpression {
return dataType;
}
- public void setDataType(DataType dataType) {
- this.dataType = dataType;
- }
-
@Override public ExpressionResult evaluate(RowIntf value) {
return new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
index dad3975..8b060e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
@@ -17,62 +17,29 @@
package org.apache.carbondata.core.scan.expression;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
public class FilterModificationNode {
private Expression currentExp;
private Expression parentExp;
private ExpressionType expType;
- private ExpressionResult literalValue;
- private String columnName;
- public FilterModificationNode(Expression currentNode, Expression parentNode,
- ExpressionType expType, DataType dataType, Object literalVal, String colName) {
+ FilterModificationNode(Expression currentNode, Expression parentNode, ExpressionType expType) {
this.currentExp = currentNode;
this.parentExp = parentNode;
this.expType = expType;
- this.columnName = colName;
- this.literalValue = new ExpressionResult(dataType, literalVal);
}
public Expression getCurrentExp() {
return currentExp;
}
- public void setCurrentExp(Expression currentExp) {
- this.currentExp = currentExp;
- }
-
public Expression getParentExp() {
return parentExp;
}
- public void setParentExp(Expression parentExp) {
- this.parentExp = parentExp;
- }
-
public ExpressionType getExpType() {
return expType;
}
- public void setExpType(ExpressionType expType) {
- this.expType = expType;
- }
-
- public ExpressionResult getLiteralValue() {
- return literalValue;
- }
-
- public void setLiteralValue(ExpressionResult literalValue) {
- this.literalValue = literalValue;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
index e7cc1c3..585bf60 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -48,7 +48,6 @@ public class RangeExpressionEvaluator {
LogServiceFactory.getLogService(RangeExpressionEvaluator.class.getName());
private Expression expr;
private Expression srcNode;
- private Expression srcParentNode;
private Expression tarNode;
private Expression tarParentNode;
@@ -64,15 +63,15 @@ public class RangeExpressionEvaluator {
this.expr = expr;
}
- public Expression getSrcNode() {
+ private Expression getSrcNode() {
return srcNode;
}
- public void setTarNode(Expression expr) {
+ private void setTarNode(Expression expr) {
this.tarNode = expr;
}
- public void setTarParentNode(Expression expr) {
+ private void setTarParentNode(Expression expr) {
this.tarParentNode = expr;
}
@@ -237,12 +236,10 @@ public class RangeExpressionEvaluator {
private void addFilterExpressionMap(Map<String, List<FilterModificationNode>> filterExpressionMap,
Expression currentNode, Expression parentNode) {
String colName = getColumnName(currentNode);
- DataType dataType = getLiteralDataType(currentNode);
- Object literalVal = getLiteralValue(currentNode);
ExpressionType expType = getExpressionType(currentNode);
FilterModificationNode filterExpression =
- new FilterModificationNode(currentNode, parentNode, expType, dataType, literalVal, colName);
+ new FilterModificationNode(currentNode, parentNode, expType);
if (null == filterExpressionMap.get(colName)) {
filterExpressionMap.put(colName, new ArrayList<FilterModificationNode>());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 3e23aa3..b882b51 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -105,8 +105,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
*
*/
public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
- FilterResolverIntf filterResolver, AbstractIndex tableSegment,
- AbsoluteTableIdentifier tableIdentifier) {
+ FilterResolverIntf filterResolver, AbstractIndex tableSegment) {
// Need to get the current dimension tables
List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
// getting the start and end index key based on filter for hitting the
@@ -401,7 +400,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
case FALSE:
return new FalseConditionalResolverImpl(expression, false, false, tableIdentifier);
case TRUE:
- return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
+ return new TrueConditionalResolverImpl(expression, false, false);
case EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
// check for implicit column in the expression
@@ -410,7 +409,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
currentCondExpression.getColumnList().get(0).getCarbonColumn();
if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) {
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
}
@@ -436,7 +434,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
// getting new dim index.
@@ -462,14 +459,12 @@ public class FilterExpressionProcessor implements FilterProcessor {
}
}
return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier,
currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
break;
case RANGE:
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, false);
case NOT_EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
@@ -492,8 +487,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, true);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, true);
}
if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
@@ -516,11 +510,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, false);
}
break;
@@ -534,7 +526,7 @@ public class FilterExpressionProcessor implements FilterProcessor {
.hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
.getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY))
|| (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure())) {
- return new ConditionalFilterResolverImpl(expression, true, true, tableIdentifier,
+ return new ConditionalFilterResolverImpl(expression, true, true,
condExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
index dc392ba..dbb6eb7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
@@ -53,7 +53,7 @@ public interface FilterProcessor {
* @return list of DataRefNode.
*/
List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
- AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier);
+ AbstractIndex segmentIndexBuilder);
/**
* This API will get the map of required partitions.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 8dcac30..0b72e77 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -49,7 +49,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -172,10 +172,8 @@ public final class FilterUtil {
segmentProperties);
case RANGE:
return new RangeValueFilterExecuterImpl(
- ((ConditionalFilterResolverImpl) filterExpressionResolverTree)
- .getDimColResolvedFilterInfo(),
- null, filterExpressionResolverTree.getFilterExpression(),
- ((ConditionalFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(),
+ filterExpressionResolverTree.getFilterExpression(),
((ConditionalFilterResolverImpl) filterExpressionResolverTree)
.getFilterRangeValues(segmentProperties), segmentProperties);
case TRUE:
@@ -551,10 +549,9 @@ public final class FilterUtil {
sortFilterModelMembers(columnExpression, evaluateResultList);
getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
filterInfo =
- getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ getFilterValues(forwardDictionary, isIncludeFilter, surrogates);
if (filterInfo.isOptimized()) {
- return getDimColumnFilterInfoAfterApplyingCBO(columnExpression,
- forwardDictionary, filterInfo);
+ return getDimColumnFilterInfoAfterApplyingCBO(forwardDictionary, filterInfo);
}
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
@@ -566,22 +563,20 @@ public final class FilterUtil {
* Method will prepare the dimfilterinfo instance by resolving the filter
* expression value to its respective surrogates.
*
- * @param columnExpression
* @param forwardDictionary
* @param isIncludeFilter
* @param filterInfo
* @param surrogates
* @return
*/
- private static ColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
- Dictionary forwardDictionary, boolean isIncludeFilter, ColumnFilterInfo filterInfo,
- List<Integer> surrogates) throws QueryExecutionException {
+ private static ColumnFilterInfo getFilterValues(Dictionary forwardDictionary,
+ boolean isIncludeFilter, List<Integer> surrogates) {
// Default value has to be added
if (surrogates.isEmpty()) {
surrogates.add(0);
}
boolean isExcludeFilterNeedsToApply = false;
- if (null == filterInfo && isIncludeFilter) {
+ if (isIncludeFilter) {
isExcludeFilterNeedsToApply =
isExcludeFilterNeedsToApply(forwardDictionary, surrogates.size());
}
@@ -593,17 +588,10 @@ public final class FilterUtil {
columnFilterInfo.setOptimized(true);
}
columnFilterInfo.setIncludeFilter(isIncludeFilter);
- if (null != filterInfo) {
- filterInfo.setIncludeFilter(isIncludeFilter);
- filterInfo.setOptimized(true);
- filterInfo.setExcludeFilterList(surrogates);
- return filterInfo;
+ if (!isIncludeFilter) {
+ columnFilterInfo.setExcludeFilterList(surrogates);
} else {
- if (!isIncludeFilter) {
- columnFilterInfo.setExcludeFilterList(surrogates);
- } else {
- columnFilterInfo.setFilterList(surrogates);
- }
+ columnFilterInfo.setFilterList(surrogates);
}
}
return columnFilterInfo;
@@ -619,8 +607,7 @@ public final class FilterUtil {
}
private static ColumnFilterInfo getDimColumnFilterInfoAfterApplyingCBO(
- ColumnExpression columnExpression, Dictionary forwardDictionary,
- ColumnFilterInfo filterInfo) throws FilterUnsupportedException, QueryExecutionException {
+ Dictionary forwardDictionary, ColumnFilterInfo filterInfo) throws FilterUnsupportedException {
List<Integer> excludeMemberSurrogates =
prepareExcludeFilterMembers(forwardDictionary, filterInfo.getFilterList());
filterInfo.setExcludeFilterList(excludeMemberSurrogates);
@@ -730,14 +717,12 @@ public final class FilterUtil {
prepareIncludeFilterMembers(expression, columnExpression, isIncludeFilter, forwardDictionary,
surrogates);
ColumnFilterInfo filterInfo =
- getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ getFilterValues(forwardDictionary, isIncludeFilter, surrogates);
if (filterInfo.isOptimized()) {
- return getDimColumnFilterInfoAfterApplyingCBO(columnExpression, forwardDictionary,
+ return getDimColumnFilterInfoAfterApplyingCBO(forwardDictionary,
filterInfo);
}
return filterInfo;
- } catch (QueryExecutionException e) {
- throw new FilterUnsupportedException(e.getMessage());
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
}
@@ -761,21 +746,16 @@ public final class FilterUtil {
* expression value to its respective surrogates in the scenario of restructure.
*
* @param expression
- * @param columnExpression
* @param defaultValues
* @param defaultSurrogate
* @return
* @throws FilterUnsupportedException
*/
- public static ColumnFilterInfo getFilterListForRS(Expression expression,
- ColumnExpression columnExpression, String defaultValues, int defaultSurrogate)
- throws FilterUnsupportedException {
+ public static ColumnFilterInfo getFilterListForRS(Expression expression, String defaultValues,
+ int defaultSurrogate) throws FilterUnsupportedException {
List<Integer> filterValuesList = new ArrayList<Integer>(20);
ColumnFilterInfo columnFilterInfo = null;
- // List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
List<String> evaluateResultListFinal = new ArrayList<String>(20);
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
try {
List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
for (ExpressionResult result : evaluateResultList) {
@@ -821,8 +801,6 @@ public final class FilterUtil {
List<String> evaluateResultListFinal = new ArrayList<String>(20);
ColumnFilterInfo columnFilterInfo = null;
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
try {
RowIntf row = new RowImpl();
if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
@@ -1775,14 +1753,14 @@ public final class FilterUtil {
* This method will compare the selected data against null values and
* flip the bitSet if any null value is found
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param bitSet
*/
- public static void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk,
+ public static void removeNullValues(DimensionColumnPage dimensionColumnPage,
BitSet bitSet, byte[] defaultValue) {
if (!bitSet.isEmpty()) {
for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- if (dimensionColumnDataChunk.compareTo(i, defaultValue) == 0) {
+ if (dimensionColumnPage.compareTo(i, defaultValue) == 0) {
bitSet.flip(i);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
index 3742e7e..214bd9d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.types.DataType;
@@ -45,7 +45,7 @@ public interface GenericQueryType {
DataType getSchemaType();
- void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) throws IOException;
+ void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder) throws IOException;
Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index f882162..e309451 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilterExecutor {
@@ -35,18 +35,18 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, useBitsetPipeLine);
if (leftFilters.isEmpty()) {
return leftFilters;
}
- BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
+ BitSetGroup rightFilter = rightExecuter.applyFilter(rawBlockletColumnChunks, useBitsetPipeLine);
if (rightFilter.isEmpty()) {
return rightFilter;
}
leftFilters.and(rightFilter);
- blockChunkHolder.setBitSetGroup(leftFilters);
+ rawBlockletColumnChunks.setBitSetGroup(leftFilters);
return leftFilters;
}
@@ -69,9 +69,10 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
return leftFilters;
}
- @Override public void readBlocks(BlocksChunkHolder blocksChunkHolder) throws IOException {
- leftExecuter.readBlocks(blocksChunkHolder);
- rightExecuter.readBlocks(blocksChunkHolder);
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ leftExecuter.readColumnChunks(rawBlockletColumnChunks);
+ rightExecuter.readColumnChunks(rawBlockletColumnChunks);
}
@Override
@@ -93,8 +94,7 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
rightFilter = ((ImplicitColumnFilterExecutor) rightExecuter)
.isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
} else {
- rightFilter = rightExecuter
- .isScanRequired(maxValue, minValue);
+ rightFilter = rightExecuter.isScanRequired(maxValue, minValue);
}
if (rightFilter.isEmpty()) {
return rightFilter;
[03/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 94a041a..b74c279 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -378,7 +378,6 @@ public abstract class AbstractDataFileFooterConverter {
cardinality[i] = segmentInfo.getColumn_cardinalities().get(i);
}
info.setColumnCardinality(cardinality);
- info.setNumberOfColumns(segmentInfo.getNum_cols());
return info;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 52305bd..0cc783e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -48,10 +48,10 @@ import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
@@ -82,7 +82,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
@@ -248,16 +248,13 @@ public final class CarbonUtil {
public static ColumnGroupModel getColGroupModel(int[][] columnGroups) {
int[] columnSplit = new int[columnGroups.length];
int noOfColumnStore = columnSplit.length;
- boolean[] columnarStore = new boolean[noOfColumnStore];
for (int i = 0; i < columnGroups.length; i++) {
columnSplit[i] = columnGroups[i].length;
- columnarStore[i] = columnGroups[i].length <= 1;
}
ColumnGroupModel colGroupModel = new ColumnGroupModel();
colGroupModel.setNoOfColumnStore(noOfColumnStore);
colGroupModel.setColumnSplit(columnSplit);
- colGroupModel.setColumnarStore(columnarStore);
colGroupModel.setColumnGroup(columnGroups);
return colGroupModel;
}
@@ -418,7 +415,7 @@ public final class CarbonUtil {
}
}
- public static int getFirstIndexUsingBinarySearch(DimensionColumnDataChunk dimColumnDataChunk,
+ public static int getFirstIndexUsingBinarySearch(DimensionColumnPage dimColumnDataChunk,
int low, int high, byte[] compareValue, boolean matchUpLimit) {
int cmpResult = 0;
while (high >= low) {
@@ -457,7 +454,7 @@ public final class CarbonUtil {
* @return the compareValue's range index in the dimColumnDataChunk
*/
public static int[] getRangeIndexUsingBinarySearch(
- DimensionColumnDataChunk dimColumnDataChunk, int low, int high, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, int low, int high, byte[] compareValue) {
int[] rangeIndex = new int[2];
int cmpResult = 0;
@@ -551,7 +548,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextLesserValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue) {
while (currentIndex - 1 >= 0
&& dimColumnDataChunk.compareTo(currentIndex - 1, compareValue) >= 0) {
--currentIndex;
@@ -571,7 +568,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextGreaterValueToTarget(int currentIndex,
- DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
+ DimensionColumnPage dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
while (currentIndex + 1 < numerOfRows
&& dimColumnDataChunk.compareTo(currentIndex + 1, compareValue) <= 0) {
++currentIndex;
@@ -934,7 +931,7 @@ public final class CarbonUtil {
return false;
}
- public static boolean[] getDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -943,7 +940,7 @@ public final class CarbonUtil {
return dictionaryEncodingArray;
}
- public static boolean[] getDirectDictionaryEncodingArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getDirectDictionaryEncodingArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -952,7 +949,7 @@ public final class CarbonUtil {
return dictionaryEncodingArray;
}
- public static boolean[] getImplicitColumnArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getImplicitColumnArray(ProjectionDimension[] queryDimensions) {
boolean[] implicitColumnArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
implicitColumnArray[i] = queryDimensions[i].getDimension().hasEncoding(Encoding.IMPLICIT);
@@ -960,7 +957,7 @@ public final class CarbonUtil {
return implicitColumnArray;
}
- public static boolean[] getComplexDataTypeArray(QueryDimension[] queryDimensions) {
+ public static boolean[] getComplexDataTypeArray(ProjectionDimension[] queryDimensions) {
boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
dictionaryEncodingArray[i] =
@@ -1008,7 +1005,6 @@ public final class CarbonUtil {
fileFooter.setColumnInTable(schema);
SegmentInfo segmentInfo = new SegmentInfo();
segmentInfo.setColumnCardinality(detailInfo.getDimLens());
- segmentInfo.setNumberOfColumns(detailInfo.getRowCount());
fileFooter.setSegmentInfo(segmentInfo);
return fileFooter;
}
@@ -1048,7 +1044,7 @@ public final class CarbonUtil {
* @return
*/
public static long calculateMetaSize(TableBlockInfo tableBlockInfo) throws IOException {
- FileHolder fileReader = null;
+ FileReader fileReader = null;
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
long footerPointer = completeBlockLength - 8;
@@ -2183,19 +2179,19 @@ public final class CarbonUtil {
/**
* Below method will be used to check filter value is present in the data chunk or not
* @param filterValues
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param low
* @param high
* @param chunkRowIndex
* @return
*/
public static int isFilterPresent(byte[][] filterValues,
- DimensionColumnDataChunk dimensionColumnDataChunk, int low, int high, int chunkRowIndex) {
+ DimensionColumnPage dimensionColumnPage, int low, int high, int chunkRowIndex) {
int compareResult = 0;
int mid = 0;
while (low <= high) {
mid = (low + high) >>> 1;
- compareResult = dimensionColumnDataChunk.compareTo(chunkRowIndex, filterValues[mid]);
+ compareResult = dimensionColumnPage.compareTo(chunkRowIndex, filterValues[mid]);
if (compareResult < 0) {
high = mid - 1;
} else if (compareResult > 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index e61b477..d665379 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -21,7 +21,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
@@ -45,7 +45,7 @@ public class DataFileFooterConverter extends AbstractDataFileFooterConverter {
@Override public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
throws IOException {
DataFileFooter dataFileFooter = new DataFileFooter();
- FileHolder fileReader = null;
+ FileReader fileReader = null;
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
long footerPointer = completeBlockLength - 8;
@@ -123,7 +123,7 @@ public class DataFileFooterConverter extends AbstractDataFileFooterConverter {
}
@Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
- FileHolder fileReader = null;
+ FileReader fileReader = null;
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
index 74fd09a..afacc0b 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriter.java
@@ -36,17 +36,6 @@ public interface CarbonDictionaryWriter extends Closeable {
void write(String value) throws IOException;
/**
- * write method that accepts one value at a time
- * This method can be used when data is huge and memory is les. In that
- * case data can be stored to a file and an iterator can iterate over it and
- * pass one value at a time
- *
- * @param value unique dictionary value
- * @throws IOException if an I/O error occurs
- */
- void write(byte[] value) throws IOException;
-
- /**
* write method that accepts list of byte arrays as value
* This can be used when data is less, then string can be converted
* to byte array for each value and added to a list
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index 82baccc..53411e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -144,7 +144,7 @@ public class CarbonDictionaryWriterImpl implements CarbonDictionaryWriter {
* @param value unique dictionary value
* @throws IOException if an I/O error occurs
*/
- @Override public void write(byte[] value) throws IOException {
+ private void write(byte[] value) throws IOException {
if (isFirstTime) {
init();
isFirstTime = false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
index 8bf8a07..08dd791 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
@@ -87,7 +87,7 @@ public class DictionaryCacheLoaderImplTest {
return 9999;
}
};
- dictionaryCacheLoader.load(dictionaryInfo, columnIdentifier, 0L, 2L, true);
+ dictionaryCacheLoader.load(dictionaryInfo, 0L, 2L, true);
assertEquals(dictionaryInfo.getDictionaryChunks().getSize(), 4);
}
@@ -97,7 +97,7 @@ public class DictionaryCacheLoaderImplTest {
return 10000;
}
};
- dictionaryCacheLoader.load(dictionaryInfo, columnIdentifier, 0L, 2L, true);
+ dictionaryCacheLoader.load(dictionaryInfo, 0L, 2L, true);
assertEquals(dictionaryInfo.getDictionaryChunks().getSize(), 2);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
deleted file mode 100644
index 5c51c87..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastore.impl.DFSFileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-
-public class DFSFileHolderImplUnitTest {
-
- private static DFSFileHolderImpl dfsFileHolder;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- dfsFileHolder = new DFSFileHolderImpl();
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- dfsFileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = dfsFileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadLongForIoException() throws IOException {
- dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test(expected = IOException.class)
- public void testReadIntForIoException() throws IOException{
- dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadIntWithFileNameForIOException() throws IOException {
- dfsFileHolder.readInt(fileNameWithEmptyContent);
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = dfsFileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testDoubleForIoException() throws IOException {
- dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
- }
-
- @Test
- public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- dfsFileHolder.readDouble(fileName, 1L);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
new file mode 100644
index 0000000..da61a94
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastore.impl.DFSFileReaderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class DFSFileReaderImplUnitTest {
+
+ private static DFSFileReaderImpl dfsFileHolder;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ dfsFileHolder = new DFSFileReaderImpl();
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ dfsFileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = dfsFileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadLongForIoException() throws IOException {
+ dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntForIoException() throws IOException{
+ dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntWithFileNameForIOException() throws IOException {
+ dfsFileHolder.readInt(fileNameWithEmptyContent);
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = dfsFileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testDoubleForIoException() throws IOException {
+ dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test
+ public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ dfsFileHolder.readDouble(fileName, 1L);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
deleted file mode 100644
index ed50d63..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-
-public class FileHolderImplUnitTest {
-
- private static FileHolderImpl fileHolder;
- private static FileHolderImpl fileHolderWithCapacity;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- fileHolder = new FileHolderImpl();
- fileHolderWithCapacity = new FileHolderImpl(50);
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- fileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = fileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadLongForIoException() throws IOException {
- fileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadIntForIoException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = fileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = fileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileNameForIOException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent);
-
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = fileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testDoubleForIoException() throws IOException {
- fileHolder.readDouble(fileNameWithEmptyContent, 1L);
-
- }
-
- @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- try {
- fileHolder.readDouble(fileName, 1L);
- } catch (Exception e) {
- assertNull(e.getMessage());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
new file mode 100644
index 0000000..a6d3235
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileReaderImplUnitTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastore.impl.FileReaderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+
+public class FileReaderImplUnitTest {
+
+ private static FileReaderImpl fileHolder;
+ private static FileReaderImpl fileHolderWithCapacity;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ fileHolder = new FileReaderImpl();
+ fileHolderWithCapacity = new FileReaderImpl(50);
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ fileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = fileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadLongForIoException() throws IOException {
+ fileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadIntForIoException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = fileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = fileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileNameForIOException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent);
+
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = fileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testDoubleForIoException() throws IOException {
+ fileHolder.readDouble(fileNameWithEmptyContent, 1L);
+
+ }
+
+ @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ try {
+ fileHolder.readDouble(fileName, 1L);
+ } catch (Exception e) {
+ assertNull(e.getMessage());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
index 9c43553..20036ec 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
@@ -84,7 +84,7 @@ public class SegmentPropertiesTest extends TestCase {
assertTrue(true);
}
- @Test public void testBlockMetadataHasProperDimensionBlockMapping() {
+ @Test public void testBlockMetadataHasProperDimensionChunkMapping() {
Map<Integer, Integer> dimensionOrdinalToBlockMapping = new HashMap<Integer, Integer>();
dimensionOrdinalToBlockMapping.put(0, 0);
dimensionOrdinalToBlockMapping.put(1, 1);
@@ -97,7 +97,7 @@ public class SegmentPropertiesTest extends TestCase {
dimensionOrdinalToBlockMapping.put(8, 5);
dimensionOrdinalToBlockMapping.put(9, 6);
Map<Integer, Integer> dimensionOrdinalToBlockMappingActual =
- blockMetadataInfos.getDimensionOrdinalToBlockMapping();
+ blockMetadataInfos.getDimensionOrdinalToChunkMapping();
assertEquals(dimensionOrdinalToBlockMapping.size(),
dimensionOrdinalToBlockMappingActual.size());
Iterator<Entry<Integer, Integer>> iterator =
@@ -112,12 +112,12 @@ public class SegmentPropertiesTest extends TestCase {
assertTrue(true);
}
- @Test public void testBlockMetadataHasProperMeasureBlockMapping() {
+ @Test public void testBlockMetadataHasProperMeasureChunkMapping() {
Map<Integer, Integer> measureOrdinalToBlockMapping = new HashMap<Integer, Integer>();
measureOrdinalToBlockMapping.put(0, 0);
measureOrdinalToBlockMapping.put(1, 1);
Map<Integer, Integer> measureOrdinalToBlockMappingActual =
- blockMetadataInfos.getMeasuresOrdinalToBlockMapping();
+ blockMetadataInfos.getMeasuresOrdinalToChunkMapping();
assertEquals(measureOrdinalToBlockMapping.size(), measureOrdinalToBlockMappingActual.size());
Iterator<Entry<Integer, Integer>> iterator = measureOrdinalToBlockMapping.entrySet().iterator();
while (iterator.hasNext()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
index bdb83cd..54b66a6 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
@@ -37,7 +37,7 @@ import org.junit.Test;
public class ColumnGroupDimensionDataChunkTest {
- static ColumnGroupDimensionDataChunk columnGroupDimensionDataChunk;
+ static ColumnGroupDimensionColumnPage columnGroupDimensionDataChunk;
static KeyGenerator keyGenerator;
@BeforeClass public static void setup() {
@@ -56,7 +56,7 @@ public class ColumnGroupDimensionDataChunkTest {
position += keyGenerator.getKeySizeInBytes();
}
columnGroupDimensionDataChunk =
- new ColumnGroupDimensionDataChunk(data, keyGenerator.getKeySizeInBytes(), 3);
+ new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3);
}
@Test public void fillChunkDataTest() {
@@ -64,7 +64,7 @@ public class ColumnGroupDimensionDataChunkTest {
ordinals.add(1);
KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
byte[] buffer = new byte[1];
- columnGroupDimensionDataChunk.fillChunkData(buffer, 0, 1, keyStructureInfo);
+ columnGroupDimensionDataChunk.fillRawData(1, 0, buffer, keyStructureInfo);
assertEquals(buffer[0], 2);
}
@@ -81,7 +81,7 @@ public class ColumnGroupDimensionDataChunkTest {
ordinals.add(2);
KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
keyStructureInfo.setMdkeyQueryDimensionOrdinal(new int[] { 2 });
- int res = columnGroupDimensionDataChunk.fillConvertedChunkData(2, 2, row, keyStructureInfo);
+ int res = columnGroupDimensionDataChunk.fillSurrogateKey(2, 2, row, keyStructureInfo);
Assert.assertTrue(Arrays.equals(row, expected));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
index 04b7a80..f327ef6 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
@@ -30,7 +30,7 @@ import org.junit.Test;
public class FixedLengthDimensionDataChunkTest {
- static FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ static FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
static byte[] data;
@BeforeClass public static void setup() {
@@ -40,7 +40,7 @@ public class FixedLengthDimensionDataChunkTest {
int invertedIndexReverse[] = { 1, 0, 5, 7, 8 };
fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(data, invertedIndex, invertedIndexReverse, 5, 4);
+ new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4);
}
@Test public void fillChunkDataTest() {
@@ -48,7 +48,7 @@ public class FixedLengthDimensionDataChunkTest {
int[] maskByteRanges = { 1, 2, 4, 6, 5 };
keyStructureInfo.setMaskByteRanges(maskByteRanges);
keyStructureInfo.setMaxKey("1234567".getBytes());
- int res = fixedLengthDimensionDataChunk.fillChunkData(data, 0, 0, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillRawData(0, 0, data, keyStructureInfo);
int expectedResult = 4 ;
assertEquals(res, expectedResult);
}
@@ -62,7 +62,7 @@ public class FixedLengthDimensionDataChunkTest {
@Test public void fillConvertedChunkDataTest() {
int[] row = { 1, 2, 4, 6 };
KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
- int res = fixedLengthDimensionDataChunk.fillConvertedChunkData(1, 0, row, keyStructureInfo);
+ int res = fixedLengthDimensionDataChunk.fillSurrogateKey(1, 0, row, keyStructureInfo);
int expectedResult = 1;
assertEquals(res, expectedResult);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index d874037..7f7719c 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -76,9 +76,9 @@ public class BTreeBlockFinderTest extends TestCase {
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(1, findFirstBlock.nodeNumber());
+ assertEquals(1, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(1, findLastBlock.nodeNumber());
+ assertEquals(1, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() {
@@ -96,9 +96,9 @@ public class BTreeBlockFinderTest extends TestCase {
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1()
@@ -118,9 +118,9 @@ public class BTreeBlockFinderTest extends TestCase {
IndexKey key =
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
@Test public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2()
@@ -141,9 +141,9 @@ public class BTreeBlockFinderTest extends TestCase {
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(0, findFirstBlock.nodeNumber());
+ assertEquals(0, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(0, findLastBlock.nodeNumber());
+ assertEquals(0, findLastBlock.nodeIndex());
}
/**
@@ -169,9 +169,9 @@ public class BTreeBlockFinderTest extends TestCase {
new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 10001, 10001 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
- assertEquals(99, findFirstBlock.nodeNumber());
+ assertEquals(99, findFirstBlock.nodeIndex());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
- assertEquals(99, findLastBlock.nodeNumber());
+ assertEquals(99, findLastBlock.nodeIndex());
}
private List<DataFileFooter> getDataFileFooterList() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
index 85bdfd8..be91410 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -50,32 +50,32 @@ public class RawBasedResultCollectorTest {
// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
// aggregatorInfo.setMeasureDataTypes(
// new DataType[] { DataTypes.INT, DataTypes.TIMESTAMP, DataTypes.INT, DataTypes.INT });
-// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+// ProjectionMeasure queryMeasure1 = new ProjectionMeasure("QMCol1");
// queryMeasure1.setQueryOrder(1);
-// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+// ProjectionMeasure queryMeasure2 = new ProjectionMeasure("QMCol2");
// queryMeasure1.setQueryOrder(2);
-// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+// ProjectionMeasure queryMeasure3 = new ProjectionMeasure("QMCol3");
// queryMeasure1.setQueryOrder(3);
-// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+// ProjectionMeasure queryMeasure4 = new ProjectionMeasure("QMCol4");
// queryMeasure1.setQueryOrder(4);
-// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+// ProjectionDimension queryDimension1 = new ProjectionDimension("QDCol1");
// queryDimension1.setQueryOrder(1);
// ColumnSchema columnSchema = new ColumnSchema();
// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
-// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+// ProjectionDimension queryDimension2 = new ProjectionDimension("QDCol2");
// queryDimension2.setQueryOrder(2);
// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
-// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+// ProjectionDimension queryDimension3 = new ProjectionDimension("QDCol3");
// queryDimension3.setQueryOrder(3);
// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
-// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+// ProjectionDimension queryDimension4 = new ProjectionDimension("QDCol4");
// queryDimension4.setQueryOrder(4);
// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
-// blockExecutionInfo.setQueryDimensions(
-// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+// blockExecutionInfo.setProjectionDimensions(
+// new ProjectionDimension[] { queryDimension1, queryDimension2, queryDimension3,
// queryDimension4 });
-// blockExecutionInfo.setQueryMeasures(
-// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+// blockExecutionInfo.setProjectionMeasures(
+// new ProjectionMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
// blockExecutionInfo.setFixedKeyUpdateRequired(true);
// blockExecutionInfo.setMeasureInfo(aggregatorInfo);
// blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
@@ -99,7 +99,7 @@ public class RawBasedResultCollectorTest {
// return new byte[][] { { 1, 2 }, { 1, 2 } };
// }
//
-// @SuppressWarnings("unused") @Mock public ColumnPage getMeasureChunk(int ordinal) {
+// @SuppressWarnings("unused") @Mock public ColumnPage readMeasureChunk(int ordinal) {
// ColumnPage ColumnPage = new ColumnPage();
// PresenceMeta presenceMeta = new PresenceMeta();
// BitSet bitSet = new BitSet();
@@ -120,10 +120,10 @@ public class RawBasedResultCollectorTest {
// }
// };
//
-// AbstractScannedResult abstractScannedResult =
+// BlockletScannedResult abstractScannedResult =
// new NonFilterQueryScannedResult(blockExecutionInfo);
-// abstractScannedResult.setNumberOfRows(2);
-// List<Object[]> result = rawBasedResultCollector.collectData(abstractScannedResult, 2);
+// abstractScannedResult.setPageFilteredRowCount(2);
+// List<Object[]> result = rawBasedResultCollector.collectResultInRow(abstractScannedResult, 2);
// int expectedResult = 2;
// assertThat(result.size(), is(equalTo(expectedResult)));
// }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 45e86f2..e8b75b7 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentPropertiesTestUtil;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import junit.framework.TestCase;
import mockit.Mock;
@@ -45,9 +45,8 @@ public class QueryUtilTest extends TestCase {
@Test public void testGetMaskedByteRangeGivingProperMaksedByteRange() {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(0));
int[] maskedByteRange = QueryUtil
.getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
int[] expectedMaskedByteRange = { 0 };
@@ -57,11 +56,10 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
dimensions.add(dimension);
}
int[] maskedByteRange =
@@ -84,11 +82,10 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaxKeyBasedOnDimensions() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
dimensions.add(dimension);
}
byte[] maxKeyBasedOnDimensions = null;
@@ -115,10 +112,8 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetMaksedByte() {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
- dimension.setDimension(segmentProperties.getDimensions().get(0));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(0));
int[] maskedByteRange = QueryUtil
.getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
int[] maskedByte = QueryUtil
@@ -203,18 +198,16 @@ public class QueryUtilTest extends TestCase {
}
@Test public void testGetSortDimensionIndexes() {
- List<QueryDimension> sortedDimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> sortedDimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
sortedDimensions.add(dimension);
}
- List<QueryDimension> queryDimensions = new ArrayList<QueryDimension>();
+ List<ProjectionDimension> queryDimensions = new ArrayList<ProjectionDimension>();
for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
+ ProjectionDimension dimension =
+ new ProjectionDimension(segmentProperties.getDimensions().get(i));
queryDimensions.add(dimension);
}
byte[] actualValue = QueryUtil.getSortDimensionIndexes(sortedDimensions, queryDimensions);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
index 90fcb74..163580d 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -28,8 +28,8 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.junit.Test;
@@ -79,27 +79,22 @@ public class RestructureUtilTest {
List<CarbonDimension> tableComplexDimensions =
Arrays.asList(tableComplexDimension1, tableComplexDimension2);
- QueryDimension queryDimension1 = new QueryDimension("Id");
- queryDimension1.setDimension(tableBlockDimension1);
- QueryDimension queryDimension2 = new QueryDimension("Name");
- queryDimension2.setDimension(tableComplexDimension2);
- QueryDimension queryDimension3 = new QueryDimension("Address");
- queryDimension3.setDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
- QueryMeasure queryMeasure1 = new QueryMeasure("Age");
- QueryMeasure queryMeasure2 = new QueryMeasure("Salary");
- queryMeasure1.setMeasure(new CarbonMeasure(columnSchema3, 2));
- queryMeasure2.setMeasure(new CarbonMeasure(columnSchema4, 4));
- List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2);
+ ProjectionDimension queryDimension1 = new ProjectionDimension(tableBlockDimension1);
+ ProjectionDimension queryDimension2 = new ProjectionDimension(tableComplexDimension2);
+ ProjectionDimension queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
+ ProjectionMeasure queryMeasure1 = new ProjectionMeasure(new CarbonMeasure(columnSchema3, 2));
+ ProjectionMeasure queryMeasure2 = new ProjectionMeasure(new CarbonMeasure(columnSchema4, 4));
+ List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2);
- List<QueryDimension> queryDimensions =
+ List<ProjectionDimension> queryDimensions =
Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
- List<QueryDimension> result = null;
+ List<ProjectionDimension> result = null;
result = RestructureUtil
.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryDimensions,
tableBlockDimensions, tableComplexDimensions, queryMeasures.size());
List<CarbonDimension> resultDimension = new ArrayList<>(result.size());
- for (QueryDimension queryDimension : result) {
+ for (ProjectionDimension queryDimension : result) {
resultDimension.add(queryDimension.getDimension());
}
assertThat(resultDimension,
@@ -126,13 +121,10 @@ public class RestructureUtilTest {
carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
- QueryMeasure queryMeasure1 = new QueryMeasure("Id");
- queryMeasure1.setMeasure(carbonMeasure1);
- QueryMeasure queryMeasure2 = new QueryMeasure("Name");
- queryMeasure2.setMeasure(carbonMeasure2);
- QueryMeasure queryMeasure3 = new QueryMeasure("Age");
- queryMeasure3.setMeasure(carbonMeasure3);
- List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
+ ProjectionMeasure queryMeasure1 = new ProjectionMeasure(carbonMeasure1);
+ ProjectionMeasure queryMeasure2 = new ProjectionMeasure(carbonMeasure2);
+ ProjectionMeasure queryMeasure3 = new ProjectionMeasure(carbonMeasure3);
+ List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, queryMeasures,
currentBlockMeasures);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index 565da04..9c7c26c 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -328,7 +328,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
return "test";
}
};
- assertTrue(FilterUtil.getFilterListForRS(expression, columnExpression, defaultValues,
+ assertTrue(FilterUtil.getFilterListForRS(expression, defaultValues,
defaultSurrogate) instanceof ColumnFilterInfo);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
index e3ae42c..11c4980 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImplTest.java
@@ -18,12 +18,12 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.util.CarbonUtil;
public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest {
- @Override public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimColumnDataChunk,
+ @Override public BitSet setFilterdIndexToBitSetNew(DimensionColumnPage dimColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
@@ -46,7 +46,7 @@ public class ExcludeFilterExecuterImplTest extends IncludeFilterExecuterImplTest
return bitSet;
}
- @Override public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimColumnDataChunk,
+ @Override public BitSet setFilterdIndexToBitSet(DimensionColumnPage dimColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
index 29dda52..1bfa875 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
@@ -18,8 +18,8 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.util.CarbonUtil;
import org.junit.Assert;
@@ -38,22 +38,22 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
- public BitSet setFilterdIndexToBitSetNew(DimensionColumnDataChunk dimensionColumnDataChunk,
+ public BitSet setFilterdIndexToBitSetNew(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
// byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.set(i);
}
}
} else if (filterValues.length == 1) {
for (int i = 0; i < numerOfRows; i++) {
- if (dimensionColumnDataChunk.compareTo(i, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(i, filterValues[0]) == 0) {
bitSet.set(i);
}
}
@@ -62,14 +62,14 @@ public class IncludeFilterExecuterImplTest extends TestCase {
return bitSet;
}
- public BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ public BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
// byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
for (int k = 0; k < filterValues.length; k++) {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[k]) == 0) {
bitSet.set(j);
}
}
@@ -163,7 +163,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
System.out.println("dimColumnSize: " + dimColumnSize);
- FixedLengthDimensionDataChunk dimensionColumnDataChunk;
+ FixedLengthDimensionColumnPage dimensionColumnDataChunk;
DimColumnExecuterFilterInfo dim = new DimColumnExecuterFilterInfo();
byte[] dataChunk = new byte[dataChunkSize * dimColumnSize];
@@ -183,7 +183,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
dim.setFilterKeys(filterKeys);
- dimensionColumnDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunkSize, dimColumnSize);
// repeat query and compare 2 result between old code and new optimized code
@@ -194,7 +194,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
oldTime = oldTime + System.currentTimeMillis() - start;
start = System.currentTimeMillis();
- BitSet bitNew = this.setFilterdIndexToBitSetNew((FixedLengthDimensionDataChunk) dimensionColumnDataChunk, dataChunkSize,
+ BitSet bitNew = this.setFilterdIndexToBitSetNew((FixedLengthDimensionColumnPage) dimensionColumnDataChunk, dataChunkSize,
filterKeys);
newTime = newTime + System.currentTimeMillis() - start;
@@ -212,7 +212,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
- private BitSet setFilterdIndexToBitSetWithColumnIndexOld(FixedLengthDimensionDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSetWithColumnIndexOld(FixedLengthDimensionColumnPage dimensionColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -243,7 +243,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
return bitSet;
}
- private BitSet setFilterdIndexToBitSetWithColumnIndexNew(FixedLengthDimensionDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSetWithColumnIndexNew(FixedLengthDimensionColumnPage dimensionColumnDataChunk,
int numerOfRows, byte[][] filterValues) {
BitSet bitSet = new BitSet(numerOfRows);
int startIndex = 0;
@@ -281,7 +281,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
int filteredValueCnt = 800;
// column dictionary size
int dimColumnSize = 2;
- FixedLengthDimensionDataChunk dimensionColumnDataChunk;
+ FixedLengthDimensionColumnPage dimensionColumnDataChunk;
DimColumnExecuterFilterInfo dim = new DimColumnExecuterFilterInfo();
byte[] dataChunk = new byte[dataChunkSize * dimColumnSize];
@@ -303,7 +303,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
}
dim.setFilterKeys(filterKeys);
- dimensionColumnDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / dimColumnSize, dimColumnSize);
// initial to run
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
deleted file mode 100644
index 541ce48..0000000
--- a/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class BatchResultTest {
- private static BatchResult batchResult;
- private static List<Object[]> rowsList = new ArrayList(2);
-
- @BeforeClass public static void setUp() {
- batchResult = new BatchResult();
- rowsList.add(0, new Integer[] { 1, 2 });
- rowsList.add(1, new Integer[] { 3 });
- }
-
- @Test public void testNext() throws NoSuchElementException {
- BatchResult rows = new BatchResult();
- rows.setRows(rowsList);
- Object[] result = rows.next();
- Assert.assertTrue(result.equals(rowsList.get(0)));
- }
-
- @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
- BatchResult rows = new BatchResult();
- List emptyList = new ArrayList(2);
- rows.setRows(emptyList);
- rows.next();
- }
-
- @Test public void testGetRows() {
- new MockUp<BatchResult>() {
- @Mock public void $init() {
- //to be left blank
- }
- };
- BatchResult batchResult = new BatchResult();
- List<Object[]> list = batchResult.getRows();
- assertNull("Number of rows is null", list);
- }
-
- @Test public void testHasNext() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- boolean result = batchResult.hasNext();
- Assert.assertTrue(result);
- }
-
- @Test public void testGetRawRow() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- Object[] actualValue = batchResult.getRawRow(0);
- Assert.assertTrue(list.get(0) == actualValue);
- }
-
- @Test public void testGetSize() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- int actualValue = batchResult.getSize();
- int expectedValue = 2;
- assertEquals(expectedValue, actualValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
new file mode 100644
index 0000000..123d64e
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/result/RowBatchTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class RowBatchTest {
+ private static RowBatch rowBatch;
+ private static List<Object[]> rowsList = new ArrayList(2);
+
+ @BeforeClass public static void setUp() {
+ rowBatch = new RowBatch();
+ rowsList.add(0, new Integer[] { 1, 2 });
+ rowsList.add(1, new Integer[] { 3 });
+ }
+
+ @Test public void testNext() throws NoSuchElementException {
+ RowBatch rows = new RowBatch();
+ rows.setRows(rowsList);
+ Object[] result = rows.next();
+ Assert.assertTrue(result.equals(rowsList.get(0)));
+ }
+
+ @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
+ RowBatch rows = new RowBatch();
+ List emptyList = new ArrayList(2);
+ rows.setRows(emptyList);
+ rows.next();
+ }
+
+ @Test public void testGetRows() {
+ new MockUp<RowBatch>() {
+ @Mock public void $init() {
+ //to be left blank
+ }
+ };
+ RowBatch rowBatch = new RowBatch();
+ List<Object[]> list = rowBatch.getRows();
+ assertNull("Number of rows is null", list);
+ }
+
+ @Test public void testHasNext() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ boolean result = rowBatch.hasNext();
+ Assert.assertTrue(result);
+ }
+
+ @Test public void testGetRawRow() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ Object[] actualValue = rowBatch.getRawRow(0);
+ Assert.assertTrue(list.get(0) == actualValue);
+ }
+
+ @Test public void testGetSize() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ rowBatch.setRows(list);
+ int actualValue = rowBatch.getSize();
+ int expectedValue = 2;
+ assertEquals(expectedValue, actualValue);
+ }
+
+}
[39/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/docs/datamap-developer-guide.md
----------------------------------------------------------------------
diff --git a/docs/datamap-developer-guide.md b/docs/datamap-developer-guide.md
new file mode 100644
index 0000000..31afd34
--- /dev/null
+++ b/docs/datamap-developer-guide.md
@@ -0,0 +1,16 @@
+# DataMap Developer Guide
+
+### Introduction
+DataMap is a data structure that can be used to accelerate certain query of the table. Different DataMap can be implemented by developers.
+Currently, there are two 2 types of DataMap supported:
+1. IndexDataMap: DataMap that leveraging index to accelerate filter query
+2. MVDataMap: DataMap that leveraging Materialized View to accelerate olap style query, like SPJG query (select, predicate, join, groupby)
+
+### DataMap provider
+When user issues `CREATE DATAMAP dm ON TABLE main USING 'provider'`, the corresponding DataMapProvider implementation will be created and initialized.
+Currently, the provider string can be:
+1. preaggregate: one type of MVDataMap that do pre-aggregate of single table
+2. timeseries: one type of MVDataMap that do pre-aggregate based on time dimension of the table
+3. class name IndexDataMapFactory implementation: Developer can implement new type of IndexDataMap by extending IndexDataMapFactory
+
+When user issues `DROP DATAMAP dm ON TABLE main`, the corresponding DataMapProvider interface will be called.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 3bc4547..007ba2f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -43,8 +43,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
@@ -756,7 +755,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
DistributableDataMapFormat datamapDstr =
new DistributableDataMapFormat(absoluteTableIdentifier, dataMapExprWrapper,
segmentIds, partitionsToPrune,
- BlockletDataMapFactory.class.getName());
+ BlockletIndexDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
// Apply expression on the blocklets.
prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index b1962c1..f208c92 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -286,7 +286,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
| GROUP BY dob,name
""".stripMargin)
}
- assert(e.getMessage.contains(s"$timeSeries keyword missing"))
+ assert(e.getMessage.contains("Only 'path' dmproperty is allowed for this datamap"))
sql("DROP TABLE IF EXISTS maintabletime")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 0f59949..6f78285 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -303,8 +303,8 @@ test("test PreAggregate table selection with timeseries and normal together") {
| GROUP BY dob,name
""".stripMargin)
- val df = sql("SELECT timeseries(dob,'year') FROM maintabletime GROUP BY timeseries(dob,'year')")
- preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
+ val df = sql("SELECT timeseries(dob,'year') FROM maintabletime GROUP BY timeseries(dob,'year')")
+ preAggTableValidator(df.queryExecution.analyzed, "maintabletime_agg1_year")
sql("DROP TABLE IF EXISTS maintabletime")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index ec76b37..efe34c6 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -201,7 +201,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("DataMap class 'abc' not found"))
+ assert(e.getMessage.equals("DataMap 'abc' not found"))
}
test("test timeseries create table 12: USING and catch MalformedCarbonCommandException") {
@@ -216,7 +216,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("DataMap class 'abc' not found"))
+ assert(e.getMessage.equals("DataMap 'abc' not found"))
}
test("test timeseries create table 13: Only one granularity level can be defined 1") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
deleted file mode 100644
index 041a63a..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datastore.FileReader
-import org.apache.carbondata.core.datastore.block.SegmentProperties
-import org.apache.carbondata.core.datastore.compression.SnappyCompressor
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
-import org.apache.carbondata.core.util.ByteUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
-
-class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
- var identifier: AbsoluteTableIdentifier = _
- var dataMapSchema: DataMapSchema = _
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- this.dataMapSchema = dataMapSchema
- }
-
- /**
- * Return a new write for this datamap
- */
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new CGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
- }
-
- /**
- * Get the datamap for segmentid
- */
- override def getDataMaps(segment: Segment): java.util.List[AbstractCoarseGrainDataMap] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map {f =>
- val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
- dataMap.init(new DataMapModel(f.getCanonicalPath))
- dataMap
- }.toList.asJava
- }
-
-
- /**
- * Get datamaps for distributable object.
- */
- override def getDataMaps(
- distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = {
- val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
- val dataMap: AbstractCoarseGrainDataMap = new CGDataMap()
- dataMap.init(new DataMapModel(mapDistributable.getFilePath))
- Seq(dataMap).asJava
- }
-
- /**
- *
- * @param event
- */
- override def fireEvent(event: Event): Unit = {
- ???
- }
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
- d
- }.toList.asJava
- }
-
-
- /**
- * Clears datamap of the segment
- */
- override def clear(segment: Segment): Unit = {
-
- }
-
- /**
- * Clear all datamaps from memory
- */
- override def clear(): Unit = {
-
- }
-
- /**
- * Return metadata of this datamap
- */
- override def getMeta: DataMapMeta = {
- new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
- List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
- }
-}
-
-class CGDataMap extends AbstractCoarseGrainDataMap {
-
- var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
- var FileReader: FileReader = _
- var filePath: String = _
- val compressor = new SnappyCompressor
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- override def init(dataMapModel: DataMapModel): Unit = {
- this.filePath = dataMapModel.getFilePath
- val size = FileFactory.getCarbonFile(filePath).getSize
- FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
- val footerLen = FileReader.readInt(filePath, size-4)
- val bytes = FileReader.readByteArray(filePath, size-footerLen-4, footerLen)
- val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(in)
- maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
- }
-
- /**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- override def prune(
- filterExp: FilterResolverIntf,
- segmentProperties: SegmentProperties,
- partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
- val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
- val expression = filterExp.getFilterExpression
- getEqualToExpression(expression, buffer)
- val value = buffer.map { f =>
- f.getChildren.get(1).evaluate(null).getString
- }
- val meta = findMeta(value(0).getBytes)
- meta.map { f=>
- new Blocklet(f._1, f._2 + "")
- }.asJava
- }
-
-
- private def findMeta(value: Array[Byte]) = {
- val tuples = maxMin.filter { f =>
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
- }
- tuples
- }
-
- private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.isInstanceOf[EqualToExpression]) {
- buffer += expression
- } else {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
- }
- getEqualToExpression(f, buffer)
- }
- }
- }
- }
-
- /**
- * Clear complete index table and release memory.
- */
- override def clear() = {
- ???
- }
-
- override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
-}
-
-class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segment: Segment,
- dataWritePath: String,
- dataMapSchema: DataMapSchema)
- extends AbstractDataMapWriter(identifier, segment, dataWritePath) {
-
- var currentBlockId: String = null
- val cgwritepath = dataWritePath + "/" +
- dataMapSchema.getDataMapName + System.nanoTime() + ".datamap"
- lazy val stream: DataOutputStream = FileFactory
- .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
- val blockletList = new ArrayBuffer[Array[Byte]]()
- val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
- val compressor = new SnappyCompressor
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String): Unit = {
- currentBlockId = blockId
- }
-
- /**
- * End of block notification
- */
- override def onBlockEnd(blockId: String): Unit = {
-
- }
-
- /**
- * Start of new blocklet notification.
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletStart(blockletId: Int): Unit = {
-
- }
-
- /**
- * End of blocklet notification
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletEnd(blockletId: Int): Unit = {
- val sorted = blockletList
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
- maxMin +=
- ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
- blockletList.clear()
- }
-
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- override def onPageAdded(blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- val size = pages(0).getPageSize
- val list = new ArrayBuffer[Array[Byte]]()
- var i = 0
- while (i < size) {
- val bytes = pages(0).getBytes(i)
- val newBytes = new Array[Byte](bytes.length - 2)
- System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
- list += newBytes
- i = i + 1
- }
- // Sort based on the column data in order to create index.
- val sorted = list
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
- blockletList += sorted.head
- blockletList += sorted.last
- }
-
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish(): Unit = {
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(maxMin)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- stream.writeInt(bytes.length)
- stream.close()
- commitFile(cgwritepath)
- }
-
-
-}
-
-class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
-
- val file2 = resourcesPath + "/compaction/fil2.csv"
- override protected def beforeAll(): Unit = {
- //n should be about 5000000 of reset if size is default 1024
- val n = 150000
- CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
- sql("DROP TABLE IF EXISTS normal_test")
- sql(
- """
- | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
- }
-
- test("test cg datamap") {
- sql("DROP TABLE IF EXISTS datamap_test_cg")
- sql(
- """
- | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
- // register datamap writer
- sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
- sql("select * from normal_test where name='n502670'"))
- }
-
- test("test cg datamap with 2 datamaps ") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(s"create datamap ggdatamap1 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
- sql(s"create datamap ggdatamap2 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='city')")
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
- }
-
- override protected def afterAll(): Unit = {
- CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
- sql("DROP TABLE IF EXISTS normal_test")
- sql("DROP TABLE IF EXISTS datamap_test_cg")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
new file mode 100644
index 0000000..72a88ea
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainIndexDataMap, AbstractCoarseGrainIndexDataMapFactory}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class CGIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapSchema: DataMapSchema = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ this.dataMapSchema = dataMapSchema
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
+ new CGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segment: Segment): java.util.List[AbstractCoarseGrainDataMap] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map {f =>
+ val dataMap: AbstractCoarseGrainIndexDataMap = new CGIndexDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainIndexDataMap] = {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractCoarseGrainIndexDataMap = new CGIndexDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event): Unit = {
+ ???
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segment: Segment): Unit = {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
+ }
+}
+
+class CGIndexDataMap extends AbstractCoarseGrainIndexDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size-4)
+ val bytes = FileReader.readByteArray(filePath, size-footerLen-4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f=>
+ new Blocklet(f._1, f._2 + "")
+ }.asJava
+ }
+
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
+ }
+ tuples
+ }
+
+ private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear() = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segment: Segment,
+ dataWritePath: String,
+ dataMapSchema: DataMapSchema)
+ extends AbstractDataMapWriter(identifier, segment, dataWritePath) {
+
+ var currentBlockId: String = null
+ val cgwritepath = dataWritePath + "/" +
+ dataMapSchema.getDataMapName + System.nanoTime() + ".datamap"
+ lazy val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
+ val blockletList = new ArrayBuffer[Array[Byte]]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ maxMin +=
+ ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in IndexDataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[Array[Byte]]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += newBytes
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ blockletList += sorted.head
+ blockletList += sorted.last
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(cgwritepath)
+ }
+
+
+}
+
+class CGIndexDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test cg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ sql(
+ """
+ | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
+ // register datamap writer
+ sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ test("test cg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(s"create datamap ggdatamap1 on table datamap_test using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"create datamap ggdatamap2 on table datamap_test using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='city')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
deleted file mode 100644
index 00d13a9..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{DataFrame, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMap}
-import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.datatype.DataTypes
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.events.Event
-
-class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
-
- var identifier: AbsoluteTableIdentifier = _
-
- override def init(identifier: AbsoluteTableIdentifier,
- dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- }
-
- override def fireEvent(event: Event): Unit = ???
-
- override def clear(segment: Segment): Unit = {}
-
- override def clear(): Unit = {}
-
- override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainDataMap] = ???
-
- override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainDataMap] = ???
-
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter =
- DataMapWriterSuite.dataMapWriterC2Mock(identifier, segment, dataWritePath)
-
- override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = {
- ???
- }
-
-}
-
-class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
- def buildTestData(numRows: Int): DataFrame = {
- import sqlContext.implicits._
- sqlContext.sparkContext.parallelize(1 to numRows, 1)
- .map(x => ("a" + x, "b", x))
- .toDF("c1", "c2", "c3")
- }
-
- def dropTable(): Unit = {
- sql("DROP TABLE IF EXISTS carbon1")
- sql("DROP TABLE IF EXISTS carbon2")
- }
-
- override def beforeAll {
- dropTable()
- }
-
- test("test write datamap 2 pages") {
- sql(s"CREATE TABLE carbon1(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
- // register datamap writer
- sql(s"CREATE DATAMAP test ON TABLE carbon1 USING '${classOf[C2DataMapFactory].getName}'")
- val df = buildTestData(33000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon1")
- .option("tempCSV", "false")
- .option("sort_columns","c1")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
- assert(
- DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "blocklet end: 0"
- ))
- DataMapWriterSuite.callbackSeq = Seq()
- }
-
- test("test write datamap 2 blocklet") {
- sql(s"CREATE TABLE carbon2(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
- sql(s"CREATE DATAMAP test ON TABLE carbon2 USING '${classOf[C2DataMapFactory].getName}'")
-
- CarbonProperties.getInstance()
- .addProperty("carbon.blockletgroup.size.in.mb", "1")
- CarbonProperties.getInstance()
- .addProperty("carbon.number.of.cores.while.loading",
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
-
- val df = buildTestData(300000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon2")
- .option("tempCSV", "false")
- .option("sort_columns","c1")
- .option("SORT_SCOPE","GLOBAL_SORT")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
- // corrected test case the min "carbon.blockletgroup.size.in.mb" size could not be less than
- // 64 MB
- assert(
- DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "add page data: blocklet 0, page 2",
- "add page data: blocklet 0, page 3",
- "add page data: blocklet 0, page 4",
- "add page data: blocklet 0, page 5",
- "add page data: blocklet 0, page 6",
- "add page data: blocklet 0, page 7",
- "add page data: blocklet 0, page 8",
- "add page data: blocklet 0, page 9",
- "blocklet end: 0"
- ))
- DataMapWriterSuite.callbackSeq = Seq()
- }
-
- override def afterAll {
- dropTable()
- }
-}
-
-object DataMapWriterSuite {
-
- var callbackSeq: Seq[String] = Seq[String]()
-
- def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segment: Segment,
- dataWritePath: String) =
- new AbstractDataMapWriter(identifier, segment, dataWritePath) {
-
- override def onPageAdded(
- blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- assert(pages.length == 1)
- assert(pages(0).getDataType == DataTypes.STRING)
- val bytes: Array[Byte] = pages(0).getByteArrayPage()(0)
- assert(bytes.sameElements(Seq(0, 1, 'b'.toByte)))
- callbackSeq :+= s"add page data: blocklet $blockletId, page $pageId"
- }
-
- override def onBlockletEnd(blockletId: Int): Unit = {
- callbackSeq :+= s"blocklet end: $blockletId"
- }
-
- override def onBlockEnd(blockId: String): Unit = {
- callbackSeq :+= s"block end $blockId"
- }
-
- override def onBlockletStart(blockletId: Int): Unit = {
- callbackSeq :+= s"blocklet start $blockletId"
- }
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String) = {
- callbackSeq :+= s"block start $blockId"
- }
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish() = {
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
deleted file mode 100644
index 90f0972..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainDataMap, AbstractFineGrainDataMapFactory}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datastore.FileReader
-import org.apache.carbondata.core.datastore.block.SegmentProperties
-import org.apache.carbondata.core.datastore.compression.SnappyCompressor
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.{Blocklet, FineGrainBlocklet, PartitionSpec}
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
-import org.apache.carbondata.core.util.ByteUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
-
-class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
- var identifier: AbsoluteTableIdentifier = _
- var dataMapSchema: DataMapSchema = _
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- this.dataMapSchema = dataMapSchema
- }
-
- /**
- * Return a new write for this datamap
- */
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new FGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
- }
-
- /**
- * Get the datamap for segmentid
- */
- override def getDataMaps(segment: Segment): java.util.List[AbstractFineGrainDataMap] = {
- val file = FileFactory
- .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val dataMap: AbstractFineGrainDataMap = new FGDataMap()
- dataMap.init(new DataMapModel(f.getCanonicalPath))
- dataMap
- }.toList.asJava
- }
-
- /**
- * Get datamap for distributable object.
- */
- override def getDataMaps(
- distributable: DataMapDistributable): java.util.List[AbstractFineGrainDataMap]= {
- val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
- val dataMap: AbstractFineGrainDataMap = new FGDataMap()
- dataMap.init(new DataMapModel(mapDistributable.getFilePath))
- Seq(dataMap).asJava
- }
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
- d
- }.toList.asJava
- }
-
- /**
- *
- * @param event
- */
- override def fireEvent(event: Event):Unit = {
- ???
- }
-
- /**
- * Clears datamap of the segment
- */
- override def clear(segment: Segment): Unit = {
- }
-
- /**
- * Clear all datamaps from memory
- */
- override def clear(): Unit = {
- }
-
- /**
- * Return metadata of this datamap
- */
- override def getMeta: DataMapMeta = {
- new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
- List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
- }
-}
-
-class FGDataMap extends AbstractFineGrainDataMap {
-
- var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
- var FileReader: FileReader = _
- var filePath: String = _
- val compressor = new SnappyCompressor
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- override def init(dataMapModel: DataMapModel): Unit = {
- this.filePath = dataMapModel.getFilePath
- val size = FileFactory.getCarbonFile(filePath).getSize
- FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
- val footerLen = FileReader.readInt(filePath, size - 4)
- val bytes = FileReader.readByteArray(filePath, size - footerLen - 4, footerLen)
- val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(in)
- maxMin = obj.readObject()
- .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
- }
-
- /**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- override def prune(
- filterExp: FilterResolverIntf,
- segmentProperties: SegmentProperties,
- partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
- val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
- val expression = filterExp.getFilterExpression
- getEqualToExpression(expression, buffer)
- val value = buffer.map { f =>
- f.getChildren.get(1).evaluate(null).getString
- }
- val meta = findMeta(value(0).getBytes)
- meta.map { f =>
- readAndFindData(f, value(0).getBytes())
- }.filter(_.isDefined).map(_.get).asJava
- }
-
- private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
- value: Array[Byte]): Option[Blocklet] = {
- val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
- val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(outputStream)
- val blockletsData = obj.readObject()
- .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
-
- import scala.collection.Searching._
- val searching = blockletsData
- .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
- (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
- override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
- y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
- }
- })
- if (searching.insertionPoint >= 0) {
- val f = blockletsData(searching.insertionPoint)
- val pages = f._3.zipWithIndex.map { p =>
- val pg = new FineGrainBlocklet.Page
- pg.setPageId(p._1)
- pg.setRowId(f._2(p._2).toArray)
- pg
- }
- Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
- } else {
- None
- }
- }
-
- private def findMeta(value: Array[Byte]) = {
- val tuples = maxMin.filter { f =>
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
- }
- tuples
- }
-
- def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.isInstanceOf[EqualToExpression]) {
- buffer += expression
- } else {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
- }
- getEqualToExpression(f, buffer)
- }
- }
- }
- }
-
- /**
- * Clear complete index table and release memory.
- */
- override def clear():Unit = {
- ???
- }
-
- override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
-}
-
-class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segment: Segment, dataWriterPath: String, dataMapSchema: DataMapSchema)
- extends AbstractDataMapWriter(identifier, segment, dataWriterPath) {
-
- var currentBlockId: String = null
- val fgwritepath = dataWriterPath + "/" + dataMapSchema.getDataMapName + System.nanoTime() +
- ".datamap"
- val stream: DataOutputStream = FileFactory
- .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
- val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
- val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
- var position: Long = 0
- val compressor = new SnappyCompressor
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String): Unit = {
- currentBlockId = blockId
- }
-
- /**
- * End of block notification
- */
- override def onBlockEnd(blockId: String): Unit = {
-
- }
-
- /**
- * Start of new blocklet notification.
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletStart(blockletId: Int): Unit = {
-
- }
-
- /**
- * End of blocklet notification
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletEnd(blockletId: Int): Unit = {
- val sorted = blockletList
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
- var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
- var addedLast: Boolean = false
- val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
- // Merge all same column values to single row.
- sorted.foreach { f =>
- if (oldValue != null) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
- oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
- addedLast = false
- } else {
- blockletListUpdated += oldValue
- oldValue = (f._1, Seq(f._2), f._3)
- addedLast = true
- }
- } else {
- oldValue = (f._1, Seq(f._2), f._3)
- addedLast = false
- }
- }
- if (!addedLast && oldValue != null) {
- blockletListUpdated += oldValue
- }
-
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(blockletListUpdated)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- maxMin +=
- ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
- ._1), position, bytes.length))
- position += bytes.length
- blockletList.clear()
- }
-
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- override def onPageAdded(blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- val size = pages(0).getPageSize
- val list = new ArrayBuffer[(Array[Byte], Int)]()
- var i = 0
- while (i < size) {
- val bytes = pages(0).getBytes(i)
- val newBytes = new Array[Byte](bytes.length - 2)
- System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
- list += ((newBytes, i))
- i = i + 1
- }
- // Sort based on the column data in order to create index.
- val sorted = list
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
- var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
- var addedLast: Boolean = false
- // Merge all same column values to single row.
- sorted.foreach { f =>
- if (oldValue != null) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
- oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
- addedLast = false
- } else {
- blockletList += oldValue
- oldValue = (f._1, Seq(f._2), Seq(pageId))
- addedLast = true
- }
- } else {
- oldValue = (f._1, Seq(f._2), Seq(pageId))
- addedLast = false
- }
- }
- if (!addedLast && oldValue != null) {
- blockletList += oldValue
- }
- }
-
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish(): Unit = {
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(maxMin)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- stream.writeInt(bytes.length)
- stream.close()
- commitFile(fgwritepath)
- }
-
-
-}
-
-class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
-
- val file2 = resourcesPath + "/compaction/fil2.csv"
-
- override protected def beforeAll(): Unit = {
- //n should be about 5000000 of reset if size is default 1024
- val n = 150000
- CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
- sql("DROP TABLE IF EXISTS normal_test")
- sql(
- """
- | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
- }
-
- test("test fg datamap") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(
- s"""
- | CREATE DATAMAP ggdatamap ON TABLE datamap_test
- | USING '${classOf[FGDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='name')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670'"),
- sql("select * from normal_test where name='n502670'"))
- }
-
- test("test fg datamap with 2 datamaps ") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(
- s"""
- | CREATE DATAMAP ggdatamap1 ON TABLE datamap_test
- | USING '${classOf[FGDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='name')
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ggdatamap2 ON TABLE datamap_test
- | USING '${classOf[FGDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='city')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
- }
-
- override protected def afterAll(): Unit = {
- CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
- sql("DROP TABLE IF EXISTS normal_test")
- sql("DROP TABLE IF EXISTS datamap_test")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
new file mode 100644
index 0000000..d92efe7
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainIndexDataMap, AbstractFineGrainIndexDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.{Blocklet, FineGrainBlocklet, PartitionSpec}
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class FGIndexDataMapFactory extends AbstractFineGrainIndexDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapSchema: DataMapSchema = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ this.dataMapSchema = dataMapSchema
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
+ new FGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segment: Segment): java.util.List[AbstractFineGrainIndexDataMap] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val dataMap: AbstractFineGrainIndexDataMap = new FGIndexDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+ /**
+ * Get datamap for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[AbstractFineGrainIndexDataMap]= {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: AbstractFineGrainIndexDataMap = new FGIndexDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event):Unit = {
+ ???
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segment: Segment): Unit = {
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
+ }
+}
+
+class FGIndexDataMap extends AbstractFineGrainIndexDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size - 4)
+ val bytes = FileReader.readByteArray(filePath, size - footerLen - 4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f =>
+ readAndFindData(f, value(0).getBytes())
+ }.filter(_.isDefined).map(_.get).asJava
+ }
+
+ private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
+ value: Array[Byte]): Option[Blocklet] = {
+ val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
+ val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(outputStream)
+ val blockletsData = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
+
+ import scala.collection.Searching._
+ val searching = blockletsData
+ .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
+ (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
+ override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
+ y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
+ }
+ })
+ if (searching.insertionPoint >= 0) {
+ val f = blockletsData(searching.insertionPoint)
+ val pages = f._3.zipWithIndex.map { p =>
+ val pg = new FineGrainBlocklet.Page
+ pg.setPageId(p._1)
+ pg.setRowId(f._2(p._2).toArray)
+ pg
+ }
+ Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
+ } else {
+ None
+ }
+ }
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
+ }
+ tuples
+ }
+
+ def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear():Unit = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segment: Segment, dataWriterPath: String, dataMapSchema: DataMapSchema)
+ extends AbstractDataMapWriter(identifier, segment, dataWriterPath) {
+
+ var currentBlockId: String = null
+ val fgwritepath = dataWriterPath + "/" + dataMapSchema.getDataMapName + System.nanoTime() +
+ ".datamap"
+ val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
+ val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
+ var position: Long = 0
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
+ var addedLast: Boolean = false
+ val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
+ addedLast = false
+ } else {
+ blockletListUpdated += oldValue
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletListUpdated += oldValue
+ }
+
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(blockletListUpdated)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ maxMin +=
+ ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
+ ._1), position, bytes.length))
+ position += bytes.length
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in IndexDataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[(Array[Byte], Int)]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += ((newBytes, i))
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
+ var addedLast: Boolean = false
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
+ addedLast = false
+ } else {
+ blockletList += oldValue
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletList += oldValue
+ }
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(fgwritepath)
+ }
+}
+
+class FGIndexDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test fg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap ON TABLE datamap_test
+ | USING '${classOf[FGIndexDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ test("test fg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap1 ON TABLE datamap_test
+ | USING '${classOf[FGIndexDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap2 ON TABLE datamap_test
+ | USING '${classOf[FGIndexDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='city')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
[47/54] [abbrv] carbondata git commit: [CARBONDATA-1543] Supported
DataMap chooser and expression for supporting multiple datamaps in single
query
Posted by ra...@apache.org.
[CARBONDATA-1543] Supported DataMap chooser and expression for supporting multiple datamaps in single query
This PR supports 3 features.
1.Load datamaps from the DataMapSchema which are created through DDL.
2.DataMap Chooser: It chooses the datamap out of available datamaps based on simple logic. Like if there is filter condition on column1 then for supposing 2 datamaps(1. column1 2. column1+column2) are supporting this column then we choose the datamap which has fewer columns that is the first datamap.
3.Expression support: Based on the filter expressions we convert them to the possible DataMap expressions and do apply expression on it.
For example, there are 2 datamaps available on table1
Datamap1 : column1
Datamap2 : column2
Query: select * from table1 where column1 ='a' and column2 =b
For the above query, we create datamap expression as AndDataMapExpression(Datamap1, DataMap2). So for the above query both the datamaps are included and the output of them will be applied AND condition to improve the performance
This closes #1510
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/56330ae2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/56330ae2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/56330ae2
Branch: refs/heads/master
Commit: 56330ae2fec2a47823e2584a2166bf27f3849c3f
Parents: 859d71c
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Tue Nov 21 15:49:11 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../exceptions/MetadataProcessException.java | 37 +++
.../carbondata/core/datamap/DataMapChooser.java | 284 +++++++++++++++++++
.../core/datamap/DataMapDistributable.java | 21 +-
.../core/datamap/DataMapStoreManager.java | 148 +++++++---
.../carbondata/core/datamap/TableDataMap.java | 23 +-
.../core/datamap/dev/DataMapFactory.java | 3 +-
.../datamap/dev/expr/AndDataMapExprWrapper.java | 99 +++++++
.../dev/expr/DataMapDistributableWrapper.java | 56 ++++
.../datamap/dev/expr/DataMapExprWrapper.java | 79 ++++++
.../dev/expr/DataMapExprWrapperImpl.java | 88 ++++++
.../datamap/dev/expr/OrDataMapExprWrapper.java | 96 +++++++
.../carbondata/core/datastore/TableSpec.java | 14 +-
.../carbondata/core/indexstore/Blocklet.java | 20 ++
.../core/indexstore/ExtendedBlocklet.java | 33 ++-
.../core/indexstore/FineGrainBlocklet.java | 8 +
.../blockletindex/BlockletDataMap.java | 2 -
.../blockletindex/BlockletDataMapFactory.java | 9 +-
.../conditional/StartsWithExpression.java | 72 +++++
.../scan/filter/FilterExpressionProcessor.java | 20 +-
.../core/scan/filter/intf/ExpressionType.java | 6 +-
.../statusmanager/SegmentStatusManager.java | 5 +-
.../datamap/examples/MinMaxDataMapFactory.java | 3 +-
.../hadoop/api/CarbonTableInputFormat.java | 32 ++-
.../carbondata/hadoop/api/DataMapJob.java | 2 +-
.../hadoop/api/DistributableDataMapFormat.java | 32 ++-
.../preaggregate/TestPreAggCreateCommand.scala | 6 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 5 +-
...CompactionSupportGlobalSortBigFileTest.scala | 2 +-
.../testsuite/dataload/TestLoadDataFrame.scala | 24 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 52 ++--
.../testsuite/datamap/DataMapWriterSuite.scala | 17 +-
.../testsuite/datamap/FGDataMapTestCase.scala | 68 +++--
.../testsuite/datamap/TestDataMapCommand.scala | 72 ++---
.../iud/InsertOverwriteConcurrentTest.scala | 0
.../carbondata/spark/rdd/SparkDataMapJob.scala | 6 +-
.../org/apache/spark/sql/CarbonSource.scala | 8 +-
.../spark/sql/SparkUnknownExpression.scala | 6 +-
.../datamap/CarbonCreateDataMapCommand.scala | 91 +++---
.../datamap/CarbonDropDataMapCommand.scala | 3 +-
.../CreatePreAggregateTableCommand.scala | 29 +-
.../preaaggregate/PreAggregateUtil.scala | 12 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../spark/sql/optimizer/CarbonFilters.scala | 20 +-
.../datamap/DataMapWriterListener.java | 6 +-
.../loading/DataLoadProcessBuilder.java | 2 +-
.../store/CarbonFactDataHandlerModel.java | 12 +-
46 files changed, 1316 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/common/src/main/java/org/apache/carbondata/common/exceptions/MetadataProcessException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/MetadataProcessException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/MetadataProcessException.java
new file mode 100644
index 0000000..aaeee5e
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/MetadataProcessException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown when failed to process metadata while executing
+ * carbon command
+ */
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class MetadataProcessException extends RuntimeException {
+ public MetadataProcessException(String message) {
+ super(message);
+ }
+
+ public MetadataProcessException(String message, Throwable cause) {
+ super(message + ": " + cause.getMessage(), cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
new file mode 100644
index 0000000..5155009
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datamap.dev.expr.AndDataMapExprWrapper;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapperImpl;
+import org.apache.carbondata.core.datamap.dev.expr.OrDataMapExprWrapper;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+import org.apache.carbondata.core.scan.expression.logical.OrExpression;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.TrueConditionalResolverImpl;
+
+/**
+ * This chooser does 2 jobs.
+ * 1. Based on filter expression it converts the available datamaps to datamap expression.
+ * For example, there are 2 datamaps available on table1
+ * Datamap1 : column1
+ * Datamap2 : column2
+ * Query: select * from table1 where column1 ='a' and column2 =b
+ * For the above query, we create datamap expression as AndDataMapExpression(Datamap1, DataMap2).
+ * So for the above query both the datamaps are included and the output of them will be
+ * applied AND condition to improve the performance
+ *
+ * 2. It chooses the datamap out of available datamaps based on simple logic.
+ * Like if there is filter condition on column1 then for
+ * supposing 2 datamaps(1. column1 2. column1+column2) are supporting this column then we choose
+ * the datamap which has fewer columns that is the first datamap.
+ */
+@InterfaceAudience.Internal
+public class DataMapChooser {
+
+ private static DataMapChooser INSTANCE;
+
+ private DataMapChooser() { }
+
+ public static DataMapChooser get() {
+ if (INSTANCE == null) {
+ INSTANCE = new DataMapChooser();
+ }
+ return INSTANCE;
+ }
+
+ /**
+ * Return a chosen datamap based on input filter. See {@link DataMapChooser}
+ */
+ public DataMapExprWrapper choose(CarbonTable carbonTable, FilterResolverIntf resolverIntf) {
+ if (resolverIntf != null) {
+ Expression expression = resolverIntf.getFilterExpression();
+ // First check for FG datamaps if any exist
+ List<TableDataMap> allDataMapFG =
+ DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapType.FG);
+ ExpressionTuple tuple = selectDataMap(expression, allDataMapFG);
+ if (tuple.dataMapExprWrapper == null) {
+ // Check for CG datamap
+ List<TableDataMap> allDataMapCG =
+ DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapType.CG);
+ tuple = selectDataMap(expression, allDataMapCG);
+ }
+ if (tuple.dataMapExprWrapper != null) {
+ return tuple.dataMapExprWrapper;
+ }
+ }
+ // Return the default datamap if no other datamap exists.
+ return new DataMapExprWrapperImpl(DataMapStoreManager.getInstance()
+ .getDefaultDataMap(carbonTable.getAbsoluteTableIdentifier()), resolverIntf);
+ }
+
+ private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap> allDataMap) {
+ switch (expression.getFilterExpressionType()) {
+ case AND:
+ if (expression instanceof AndExpression) {
+ AndExpression andExpression = (AndExpression) expression;
+ ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap);
+ ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap);
+ Set<ExpressionType> filterExpressionTypes = new HashSet<>();
+ // If both left and right has datamap then we can either merge both datamaps to single
+ // datamap if possible. Otherwise apply AND expression.
+ if (left.dataMapExprWrapper != null && right.dataMapExprWrapper != null) {
+ filterExpressionTypes.add(
+ left.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
+ .getFilterExpressionType());
+ filterExpressionTypes.add(
+ right.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
+ .getFilterExpressionType());
+ List<ColumnExpression> columnExpressions = new ArrayList<>();
+ columnExpressions.addAll(left.columnExpressions);
+ columnExpressions.addAll(right.columnExpressions);
+ // Check if we can merge them to single datamap.
+ TableDataMap dataMap =
+ chooseDataMap(allDataMap, columnExpressions, filterExpressionTypes);
+ if (dataMap != null) {
+ ExpressionTuple tuple = new ExpressionTuple();
+ tuple.columnExpressions = columnExpressions;
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
+ new TrueConditionalResolverImpl(expression, false, false));
+ return tuple;
+ } else {
+ // Apply AND expression.
+ ExpressionTuple tuple = new ExpressionTuple();
+ tuple.columnExpressions = columnExpressions;
+ tuple.dataMapExprWrapper =
+ new AndDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper,
+ new TrueConditionalResolverImpl(expression, false, false));
+ return tuple;
+ }
+ } else if (left.dataMapExprWrapper != null && right.dataMapExprWrapper == null) {
+ return left;
+ } else if (left.dataMapExprWrapper == null && right.dataMapExprWrapper != null) {
+ return right;
+ } else {
+ return left;
+ }
+ }
+ break;
+ case OR:
+ if (expression instanceof OrExpression) {
+ OrExpression orExpression = (OrExpression) expression;
+ ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap);
+ ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap);
+ Set<ExpressionType> filterExpressionTypes = new HashSet<>();
+ // If both left and right has datamap then we can either merge both datamaps to single
+ // datamap if possible. Otherwise apply OR expression.
+ if (left.dataMapExprWrapper != null && right.dataMapExprWrapper != null) {
+ filterExpressionTypes.add(
+ left.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
+ .getFilterExpressionType());
+ filterExpressionTypes.add(
+ right.dataMapExprWrapper.getFilterResolverIntf().getFilterExpression()
+ .getFilterExpressionType());
+ List<ColumnExpression> columnExpressions = new ArrayList<>();
+ columnExpressions.addAll(left.columnExpressions);
+ columnExpressions.addAll(right.columnExpressions);
+ TableDataMap dataMap =
+ chooseDataMap(allDataMap, columnExpressions, filterExpressionTypes);
+ if (dataMap != null) {
+ ExpressionTuple tuple = new ExpressionTuple();
+ tuple.columnExpressions = columnExpressions;
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
+ new TrueConditionalResolverImpl(expression, false, false));
+ return tuple;
+ } else {
+ ExpressionTuple tuple = new ExpressionTuple();
+ tuple.columnExpressions = columnExpressions;
+ tuple.dataMapExprWrapper =
+ new OrDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper,
+ new TrueConditionalResolverImpl(expression, false, false));
+ return tuple;
+ }
+ } else {
+ left.dataMapExprWrapper = null;
+ return left;
+ }
+ }
+ break;
+ default:
+ ExpressionTuple tuple = new ExpressionTuple();
+ extractColumnExpression(expression, tuple.columnExpressions);
+ Set<ExpressionType> filterExpressionTypes = new HashSet<>();
+ filterExpressionTypes.add(expression.getFilterExpressionType());
+ TableDataMap dataMap =
+ chooseDataMap(allDataMap, tuple.columnExpressions, filterExpressionTypes);
+ if (dataMap != null) {
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
+ new TrueConditionalResolverImpl(expression, false, false));
+ }
+ return tuple;
+ }
+ return new ExpressionTuple();
+ }
+
+ private void extractColumnExpression(Expression expression,
+ List<ColumnExpression> columnExpressions) {
+ if (expression instanceof ColumnExpression) {
+ columnExpressions.add((ColumnExpression) expression);
+ } else if (expression != null) {
+ List<Expression> children = expression.getChildren();
+ if (children != null && children.size() > 0) {
+ for (Expression exp : children) {
+ extractColumnExpression(exp, columnExpressions);
+ }
+ }
+ }
+ }
+
+ private TableDataMap chooseDataMap(List<TableDataMap> allDataMap,
+ List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) {
+ List<DataMapTuple> tuples = new ArrayList<>();
+ for (TableDataMap dataMap : allDataMap) {
+ if (contains(dataMap.getDataMapFactory().getMeta(), columnExpressions, expressionTypes)) {
+ tuples.add(
+ new DataMapTuple(dataMap.getDataMapFactory().getMeta().getIndexedColumns().size(),
+ dataMap));
+ }
+ }
+ if (tuples.size() > 0) {
+ Collections.sort(tuples);
+ return tuples.get(0).dataMap;
+ }
+ return null;
+ }
+
+ private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions,
+ Set<ExpressionType> expressionTypes) {
+ if (mapMeta.getIndexedColumns().size() == 0 || columnExpressions.size() == 0) {
+ return false;
+ }
+ boolean contains = true;
+ for (ColumnExpression expression : columnExpressions) {
+ if (!mapMeta.getIndexedColumns().contains(expression.getColumnName()) || !mapMeta
+ .getOptimizedOperation().containsAll(expressionTypes)) {
+ contains = false;
+ break;
+ }
+ }
+ return contains;
+ }
+
+ private static class ExpressionTuple {
+
+ DataMapExprWrapper dataMapExprWrapper;
+
+ List<ColumnExpression> columnExpressions = new ArrayList<>();
+
+ }
+
+ private static class DataMapTuple implements Comparable<DataMapTuple> {
+
+ int order;
+
+ TableDataMap dataMap;
+
+ public DataMapTuple(int order, TableDataMap dataMap) {
+ this.order = order;
+ this.dataMap = dataMap;
+ }
+
+ @Override public int compareTo(DataMapTuple o) {
+ return order - o.order;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ DataMapTuple that = (DataMapTuple) o;
+
+ if (order != that.order) return false;
+ return dataMap != null ? dataMap.equals(that.dataMap) : that.dataMap == null;
+ }
+
+ @Override public int hashCode() {
+ int result = order;
+ result = 31 * result + (dataMap != null ? dataMap.hashCode() : 0);
+ return result;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
index edd724a..473d8d4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.io.Serializable;
import org.apache.carbondata.core.datastore.block.Distributable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -33,11 +34,9 @@ public abstract class DataMapDistributable extends InputSplit
private Segment segment;
- private String dataMapName;
-
private String[] locations;
- private String dataMapFactoryClass;
+ private DataMapSchema dataMapSchema;
public String getTablePath() {
return tablePath;
@@ -55,20 +54,12 @@ public abstract class DataMapDistributable extends InputSplit
this.segment = segment;
}
- public String getDataMapName() {
- return dataMapName;
- }
-
- public void setDataMapName(String dataMapName) {
- this.dataMapName = dataMapName;
- }
-
- public String getDataMapFactoryClass() {
- return dataMapFactoryClass;
+ public DataMapSchema getDataMapSchema() {
+ return dataMapSchema;
}
- public void setDataMapFactoryClass(String dataMapFactoryClass) {
- this.dataMapFactoryClass = dataMapFactoryClass;
+ public void setDataMapSchema(DataMapSchema dataMapSchema) {
+ this.dataMapSchema = dataMapSchema;
}
public void setLocations(String[] locations) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index 8d80b4d..cd89d74 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -22,14 +22,19 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.carbondata.common.exceptions.MetadataProcessException;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
@@ -55,48 +60,76 @@ public final class DataMapStoreManager {
}
- public List<TableDataMap> getAllDataMap(AbsoluteTableIdentifier identifier) {
- return allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
+ /**
+ * It gives all datamaps of type @mapType except the default datamap.
+ *
+ */
+ public List<TableDataMap> getAllDataMap(CarbonTable carbonTable, DataMapType mapType) {
+ List<TableDataMap> dataMaps = new ArrayList<>();
+ List<TableDataMap> tableDataMaps = getAllDataMap(carbonTable);
+ if (tableDataMaps != null) {
+ for (TableDataMap dataMap : tableDataMaps) {
+ if (mapType == dataMap.getDataMapFactory().getDataMapType()) {
+ dataMaps.add(dataMap);
+ }
+ }
+ }
+ return dataMaps;
}
- // TODO its a temporary method till chooser is implemented
- public TableDataMap chooseDataMap(AbsoluteTableIdentifier identifier) {
- List<TableDataMap> tableDataMaps = getAllDataMap(identifier);
- if (tableDataMaps != null && tableDataMaps.size() > 0) {
- for (TableDataMap dataMap: tableDataMaps) {
- if (!dataMap.getDataMapName().equalsIgnoreCase(BlockletDataMap.NAME)) {
- return dataMap;
+ /**
+ * It gives all datamaps except the default datamap.
+ *
+ * @return
+ */
+ public List<TableDataMap> getAllDataMap(CarbonTable carbonTable) {
+ List<DataMapSchema> dataMapSchemaList = carbonTable.getTableInfo().getDataMapSchemaList();
+ List<TableDataMap> dataMaps = new ArrayList<>();
+ if (dataMapSchemaList != null) {
+ for (DataMapSchema dataMapSchema : dataMapSchemaList) {
+ if (!dataMapSchema.getClassName()
+ .equalsIgnoreCase(DataMapProvider.PREAGGREGATE.getClassName())) {
+ dataMaps.add(getDataMap(carbonTable.getAbsoluteTableIdentifier(), dataMapSchema));
}
}
- return tableDataMaps.get(0);
- } else {
- return getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
}
+ return dataMaps;
}
/**
- * Get the datamap for reading data.
+ * It gives the default datamap of the table. Default datamap of any table is BlockletDataMap
*
- * @param dataMapName
- * @param factoryClass
+ * @param identifier
* @return
*/
- public TableDataMap getDataMap(AbsoluteTableIdentifier identifier,
- String dataMapName, String factoryClass) {
+ public TableDataMap getDefaultDataMap(AbsoluteTableIdentifier identifier) {
+ return getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
+ }
+
+ /**
+ * Get the datamap for reading data.
+ */
+ public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
List<TableDataMap> tableDataMaps = allDataMaps.get(table);
- TableDataMap dataMap;
- if (tableDataMaps == null) {
+ TableDataMap dataMap = null;
+ if (tableDataMaps != null) {
+ dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableDataMaps);
+ }
+ if (dataMap == null) {
synchronized (table.intern()) {
tableDataMaps = allDataMaps.get(table);
- if (tableDataMaps == null) {
- dataMap = createAndRegisterDataMap(identifier, factoryClass, dataMapName);
- } else {
- dataMap = getTableDataMap(dataMapName, tableDataMaps);
+ if (tableDataMaps != null) {
+ dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableDataMaps);
+ }
+ if (dataMap == null) {
+ try {
+ dataMap = createAndRegisterDataMap(identifier, dataMapSchema);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
- } else {
- dataMap = getTableDataMap(dataMapName, tableDataMaps);
}
if (dataMap == null) {
@@ -110,7 +143,8 @@ public final class DataMapStoreManager {
* The datamap is created using datamap name, datamap factory class and table identifier.
*/
public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
- String factoryClassName, String dataMapName) {
+ DataMapSchema dataMapSchema)
+ throws MalformedDataMapCommandException {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
@@ -118,16 +152,20 @@ public final class DataMapStoreManager {
if (tableDataMaps == null) {
tableDataMaps = new ArrayList<>();
}
+ String dataMapName = dataMapSchema.getDataMapName();
TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
- if (dataMap != null && dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
- throw new RuntimeException("Already datamap exists in that path with type " + dataMapName);
+ if (dataMap != null && dataMap.getDataMapSchema().getDataMapName()
+ .equalsIgnoreCase(dataMapName)) {
+ throw new MalformedDataMapCommandException("Already datamap exists in that path with type " +
+ dataMapName);
}
try {
+ // try to create datamap by reflection to test whether it is a valid DataMapFactory class
Class<? extends DataMapFactory> factoryClass =
- (Class<? extends DataMapFactory>) Class.forName(factoryClassName);
+ (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
DataMapFactory dataMapFactory = factoryClass.newInstance();
- dataMapFactory.init(identifier, dataMapName);
+ dataMapFactory.init(identifier, dataMapSchema);
BlockletDetailsFetcher blockletDetailsFetcher;
SegmentPropertiesFetcher segmentPropertiesFetcher = null;
if (dataMapFactory instanceof BlockletDetailsFetcher) {
@@ -136,11 +174,14 @@ public final class DataMapStoreManager {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory, blockletDetailsFetcher,
+ dataMap = new TableDataMap(identifier, dataMapSchema, dataMapFactory, blockletDetailsFetcher,
segmentPropertiesFetcher);
- } catch (Exception e) {
- LOGGER.error(e);
- throw new RuntimeException(e);
+ } catch (ClassNotFoundException e) {
+ throw new MalformedDataMapCommandException("DataMap class '" +
+ dataMapSchema.getClassName() + "' not found");
+ } catch (Throwable e) {
+ throw new MetadataProcessException(
+ "failed to create DataMap instance for '" + dataMapSchema.getClassName() + "'", e);
}
tableDataMaps.add(dataMap);
allDataMaps.put(table, tableDataMaps);
@@ -150,8 +191,7 @@ public final class DataMapStoreManager {
private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableDataMaps) {
TableDataMap dataMap = null;
for (TableDataMap tableDataMap : tableDataMaps) {
- if (tableDataMap.getDataMapName().equals(dataMapName) || (!tableDataMap.getDataMapName()
- .equals(""))) {
+ if (tableDataMap.getDataMapSchema().getDataMapName().equals(dataMapName)) {
dataMap = tableDataMap;
break;
}
@@ -160,16 +200,30 @@ public final class DataMapStoreManager {
}
/**
+ * Clear the invalid segments from all the datamaps of the table
+ * @param carbonTable
+ * @param segments
+ */
+ public void clearInvalidSegments(CarbonTable carbonTable, List<Segment> segments) {
+ getDefaultDataMap(carbonTable.getAbsoluteTableIdentifier()).clear(segments);
+ List<TableDataMap> allDataMap = getAllDataMap(carbonTable);
+ for (TableDataMap dataMap: allDataMap) {
+ dataMap.clear(segments);
+ }
+
+ }
+
+ /**
* Clear the datamap/datamaps of a table from memory
+ *
* @param identifier Table identifier
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier) {
String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
- List<TableDataMap> tableDataMaps =
- allDataMaps.get(tableUniqueName);
+ List<TableDataMap> tableDataMaps = allDataMaps.get(tableUniqueName);
segmentRefreshMap.remove(identifier.uniqueName());
if (tableDataMaps != null) {
- for (TableDataMap tableDataMap: tableDataMaps) {
+ for (TableDataMap tableDataMap : tableDataMaps) {
if (tableDataMap != null) {
tableDataMap.clear();
break;
@@ -181,6 +235,7 @@ public final class DataMapStoreManager {
/**
* Clear the datamap/datamaps of a table from memory
+ *
* @param identifier Table identifier
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
@@ -188,8 +243,9 @@ public final class DataMapStoreManager {
allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
if (tableDataMaps != null) {
int i = 0;
- for (TableDataMap tableDataMap: tableDataMaps) {
- if (tableDataMap != null && dataMapName.equalsIgnoreCase(tableDataMap.getDataMapName())) {
+ for (TableDataMap tableDataMap : tableDataMaps) {
+ if (tableDataMap != null && dataMapName
+ .equalsIgnoreCase(tableDataMap.getDataMapSchema().getDataMapName())) {
tableDataMap.clear();
tableDataMaps.remove(i);
break;
@@ -201,17 +257,18 @@ public final class DataMapStoreManager {
/**
* Get the blocklet datamap factory to get the detail information of blocklets
+ *
* @param identifier
* @return
*/
private BlockletDetailsFetcher getBlockletDetailsFetcher(AbsoluteTableIdentifier identifier) {
- TableDataMap blockletMap =
- getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
+ TableDataMap blockletMap = getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
return (BlockletDetailsFetcher) blockletMap.getDataMapFactory();
}
/**
* Returns the singleton instance
+ *
* @return
*/
public static DataMapStoreManager getInstance() {
@@ -271,7 +328,7 @@ public final class DataMapStoreManager {
}
public void refreshSegments(List<String> segmentIds) {
- for (String segmentId: segmentIds) {
+ for (String segmentId : segmentIds) {
manualSegmentRefresh.put(segmentId, true);
}
}
@@ -286,5 +343,4 @@ public final class DataMapStoreManager {
}
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 62a9119..3c66c89 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.events.Event;
import org.apache.carbondata.events.OperationContext;
@@ -46,7 +47,7 @@ public final class TableDataMap extends OperationEventListener {
private AbsoluteTableIdentifier identifier;
- private String dataMapName;
+ private DataMapSchema dataMapSchema;
private DataMapFactory dataMapFactory;
@@ -57,11 +58,11 @@ public final class TableDataMap extends OperationEventListener {
/**
* It is called to initialize and load the required table datamap metadata.
*/
- public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+ public TableDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema,
DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
- this.dataMapName = dataMapName;
+ this.dataMapSchema = dataMapSchema;
this.dataMapFactory = dataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
this.segmentPropertiesFetcher = segmentPropertiesFetcher;
@@ -116,10 +117,9 @@ public final class TableDataMap extends OperationEventListener {
for (Segment segment : segments) {
List<DataMapDistributable> list = dataMapFactory.toDistributable(segment);
for (DataMapDistributable distributable: list) {
- distributable.setDataMapName(dataMapName);
+ distributable.setDataMapSchema(dataMapSchema);
distributable.setSegment(segment);
distributable.setTablePath(identifier.getTablePath());
- distributable.setDataMapFactoryClass(dataMapFactory.getClass().getName());
}
distributables.addAll(list);
}
@@ -148,7 +148,8 @@ public final class TableDataMap extends OperationEventListener {
}
BlockletSerializer serializer = new BlockletSerializer();
String writePath =
- identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapName;
+ identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapSchema
+ .getDataMapName();
if (dataMapFactory.getDataMapType() == DataMapType.FG) {
FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
}
@@ -183,13 +184,9 @@ public final class TableDataMap extends OperationEventListener {
public void clear() {
dataMapFactory.clear();
}
- /**
- * Get the unique name of datamap
- *
- * @return
- */
- public String getDataMapName() {
- return dataMapName;
+
+ public DataMapSchema getDataMapSchema() {
+ return dataMapSchema;
}
public DataMapFactory getDataMapFactory() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index d8a467f..b59de81 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.events.Event;
/**
@@ -34,7 +35,7 @@ public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
*/
- void init(AbsoluteTableIdentifier identifier, String dataMapName);
+ void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
/**
* Return a new write for this datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
new file mode 100644
index 0000000..2abe253
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * And expression for datamaps
+ */
+public class AndDataMapExprWrapper implements DataMapExprWrapper {
+
+ private DataMapExprWrapper left;
+
+ private DataMapExprWrapper right;
+
+ private FilterResolverIntf resolverIntf;
+
+ public AndDataMapExprWrapper(DataMapExprWrapper left, DataMapExprWrapper right,
+ FilterResolverIntf resolverIntf) {
+ this.left = left;
+ this.right = right;
+ this.resolverIntf = resolverIntf;
+ }
+
+ @Override public List<ExtendedBlocklet> prune(List<Segment> segments,
+ List<PartitionSpec> partitionsToPrune) throws IOException {
+ List<ExtendedBlocklet> leftPrune = left.prune(segments, partitionsToPrune);
+ List<ExtendedBlocklet> rightPrune = right.prune(segments, partitionsToPrune);
+ List<ExtendedBlocklet> andBlocklets = new ArrayList<>();
+ for (ExtendedBlocklet blocklet : leftPrune) {
+ if (rightPrune.contains(blocklet)) {
+ andBlocklets.add(blocklet);
+ }
+ }
+ return andBlocklets;
+ }
+
+ @Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets)
+ throws IOException {
+ List<ExtendedBlocklet> leftPrune = left.pruneBlocklets(blocklets);
+ List<ExtendedBlocklet> rightPrune = right.pruneBlocklets(blocklets);
+ List<ExtendedBlocklet> andBlocklets = new ArrayList<>();
+ for (ExtendedBlocklet blocklet : leftPrune) {
+ if (rightPrune.contains(blocklet)) {
+ andBlocklets.add(blocklet);
+ }
+ }
+ return andBlocklets;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf() {
+ return resolverIntf;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf(String uniqueId) {
+ FilterResolverIntf leftExp = left.getFilterResolverIntf(uniqueId);
+ FilterResolverIntf rightExp = right.getFilterResolverIntf(uniqueId);
+ if (leftExp != null) {
+ return leftExp;
+ } else if (rightExp != null) {
+ return rightExp;
+ }
+ return null;
+ }
+
+ @Override public List<DataMapDistributableWrapper> toDistributable(List<Segment> segments)
+ throws IOException {
+ List<DataMapDistributableWrapper> wrappers = new ArrayList<>();
+ wrappers.addAll(left.toDistributable(segments));
+ wrappers.addAll(right.toDistributable(segments));
+ return wrappers;
+ }
+
+ @Override public DataMapType getDataMapType() {
+ return left.getDataMapType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
new file mode 100644
index 0000000..9075032
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapDistributableWrapper.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+
+public class DataMapDistributableWrapper extends InputSplit implements Serializable {
+
+ private String uniqueId;
+
+ private DataMapDistributable distributable;
+
+ public DataMapDistributableWrapper(String uniqueId, DataMapDistributable distributable) {
+ this.uniqueId = uniqueId;
+ this.distributable = distributable;
+ }
+
+ public String getUniqueId() {
+ return uniqueId;
+ }
+
+ public DataMapDistributable getDistributable() {
+ return distributable;
+ }
+
+ public void setDistributable(DataMapDistributable distributable) {
+ this.distributable = distributable;
+ }
+
+ @Override public long getLength() throws IOException, InterruptedException {
+ return distributable.getLength();
+ }
+
+ @Override public String[] getLocations() throws IOException, InterruptedException {
+ return distributable.getLocations();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
new file mode 100644
index 0000000..2baba6f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * It is the wrapper around datamap and related filter expression. By using it user can apply
+ * datamaps in expression style.
+ */
+public interface DataMapExprWrapper extends Serializable {
+
+ /**
+ * It get the blocklets from each leaf node datamap and apply expressions on the blocklets
+ * using list of segments, it is used in case on non distributable datamap.
+ */
+ List<ExtendedBlocklet> prune(List<Segment> segments, List<PartitionSpec> partitionsToPrune)
+ throws IOException;
+
+ /**
+ * It is used in case on distributable datamap. First using job it gets all blockets from all
+ * related datamaps. These blocklets are passed to this method to apply expression.
+ * @param blocklets
+ * @return
+ * @throws IOException
+ */
+ List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets) throws IOException;
+
+ /**
+ * Get the underlying filter expression.
+ * @return
+ */
+ FilterResolverIntf getFilterResolverIntf();
+
+ /**
+ * Convert to distributable objects for executing job.
+ * @param segments
+ * @return
+ * @throws IOException
+ */
+ List<DataMapDistributableWrapper> toDistributable(List<Segment> segments) throws IOException;
+
+ /**
+ * Each leaf node is identified by uniqueid, so if user wants the underlying filter expression for
+ * any leaf node then this method can be used.
+ * @param uniqueId
+ * @return
+ */
+ FilterResolverIntf getFilterResolverIntf(String uniqueId);
+
+ /**
+ * Get the datamap type.
+ * @return
+ */
+ DataMapType getDataMapType();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
new file mode 100644
index 0000000..fac6cc1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+public class DataMapExprWrapperImpl implements DataMapExprWrapper {
+
+ private static final long serialVersionUID = -6240385328696074171L;
+
+ private transient TableDataMap dataMap;
+
+ private FilterResolverIntf expression;
+
+ private String uniqueId;
+
+ public DataMapExprWrapperImpl(TableDataMap dataMap, FilterResolverIntf expression) {
+ this.dataMap = dataMap;
+ this.expression = expression;
+ this.uniqueId = UUID.randomUUID().toString();
+ }
+
+ @Override public List<ExtendedBlocklet> prune(List<Segment> segments,
+ List<PartitionSpec> partitionsToPrune) throws IOException {
+ return dataMap.prune(segments, expression, partitionsToPrune);
+ }
+
+ @Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets)
+ throws IOException {
+ List<ExtendedBlocklet> blockletList = new ArrayList<>();
+ for (ExtendedBlocklet blocklet: blocklets) {
+ if (blocklet.getDataMapUniqueId().equals(uniqueId)) {
+ blockletList.add(blocklet);
+ }
+ }
+ return blockletList;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf() {
+ return expression;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf(String uniqueId) {
+ if (this.uniqueId.equals(uniqueId)) {
+ return expression;
+ }
+ return null;
+ }
+
+ @Override public List<DataMapDistributableWrapper> toDistributable(List<Segment> segments)
+ throws IOException {
+ List<DataMapDistributable> dataMapDistributables = dataMap.toDistributable(segments);
+ List<DataMapDistributableWrapper> wrappers = new ArrayList<>();
+ for (DataMapDistributable distributable : dataMapDistributables) {
+ wrappers.add(new DataMapDistributableWrapper(uniqueId, distributable));
+ }
+ return wrappers;
+ }
+
+ @Override public DataMapType getDataMapType() {
+ return dataMap.getDataMapFactory().getDataMapType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
new file mode 100644
index 0000000..3c63202
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * Or expression for datamaps
+ */
+public class OrDataMapExprWrapper implements DataMapExprWrapper {
+
+ private DataMapExprWrapper left;
+
+ private DataMapExprWrapper right;
+
+ private FilterResolverIntf resolverIntf;
+
+ public OrDataMapExprWrapper(DataMapExprWrapper left, DataMapExprWrapper right,
+ FilterResolverIntf resolverIntf) {
+ this.left = left;
+ this.right = right;
+ this.resolverIntf = resolverIntf;
+ }
+
+ @Override public List<ExtendedBlocklet> prune(List<Segment> segments,
+ List<PartitionSpec> partitionsToPrune) throws IOException {
+ List<ExtendedBlocklet> leftPrune = left.prune(segments, partitionsToPrune);
+ List<ExtendedBlocklet> rightPrune = right.prune(segments, partitionsToPrune);
+ Set<ExtendedBlocklet> andBlocklets = new HashSet<>();
+ andBlocklets.addAll(leftPrune);
+ andBlocklets.addAll(rightPrune);
+ return new ArrayList<>(andBlocklets);
+ }
+
+ @Override public List<ExtendedBlocklet> pruneBlocklets(List<ExtendedBlocklet> blocklets)
+ throws IOException {
+ List<ExtendedBlocklet> leftPrune = left.pruneBlocklets(blocklets);
+ List<ExtendedBlocklet> rightPrune = right.pruneBlocklets(blocklets);
+ Set<ExtendedBlocklet> andBlocklets = new HashSet<>();
+ andBlocklets.addAll(leftPrune);
+ andBlocklets.addAll(rightPrune);
+ return new ArrayList<>(andBlocklets);
+ }
+
+ @Override public List<DataMapDistributableWrapper> toDistributable(List<Segment> segments)
+ throws IOException {
+ List<DataMapDistributableWrapper> wrappers = new ArrayList<>();
+ wrappers.addAll(left.toDistributable(segments));
+ wrappers.addAll(right.toDistributable(segments));
+ return wrappers;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf() {
+ return resolverIntf;
+ }
+
+ @Override public FilterResolverIntf getFilterResolverIntf(String uniqueId) {
+ FilterResolverIntf leftExp = left.getFilterResolverIntf(uniqueId);
+ FilterResolverIntf rightExp = right.getFilterResolverIntf(uniqueId);
+ if (leftExp != null) {
+ return leftExp;
+ } else if (rightExp != null) {
+ return rightExp;
+ }
+ return null;
+ }
+
+
+ @Override public DataMapType getDataMapType() {
+ return left.getDataMapType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index eb36c8d..93da81e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.DecimalType;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -38,7 +39,14 @@ public class TableSpec {
// number of simple dimensions
private int numSimpleDimensions;
- public TableSpec(List<CarbonDimension> dimensions, List<CarbonMeasure> measures) {
+ private CarbonTable carbonTable;
+
+ public TableSpec(CarbonTable carbonTable) {
+ this.carbonTable = carbonTable;
+ List<CarbonDimension> dimensions =
+ carbonTable.getDimensionByTableName(carbonTable.getTableName());
+ List<CarbonMeasure> measures =
+ carbonTable.getMeasureByTableName(carbonTable.getTableName());
// first calculate total number of columnar field considering column group and complex column
numSimpleDimensions = 0;
for (CarbonDimension dimension : dimensions) {
@@ -112,6 +120,10 @@ public class TableSpec {
return measureSpec.length;
}
+ public CarbonTable getCarbonTable() {
+ return carbonTable;
+ }
+
public static class ColumnSpec implements Writable {
// field name of this column
private String fieldName;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index c731e07..052d269 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -58,4 +58,24 @@ public class Blocklet implements Writable,Serializable {
blockId = in.readUTF();
blockletId = in.readUTF();
}
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ Blocklet blocklet = (Blocklet) o;
+
+ if (blockId != null ? !blockId.equals(blocklet.blockId) : blocklet.blockId != null) {
+ return false;
+ }
+ return blockletId != null ?
+ blockletId.equals(blocklet.blockletId) :
+ blocklet.blockletId == null;
+ }
+
+ @Override public int hashCode() {
+ int result = blockId != null ? blockId.hashCode() : 0;
+ result = 31 * result + (blockletId != null ? blockletId.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
index 58a9344..d2af5cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/ExtendedBlocklet.java
@@ -29,13 +29,12 @@ public class ExtendedBlocklet extends Blocklet {
private String[] location;
- private String path;
-
private String dataMapWriterPath;
+ private String dataMapUniqueId;
+
public ExtendedBlocklet(String path, String blockletId) {
super(path, blockletId);
- this.path = path;
}
public BlockletDetailInfo getDetailInfo() {
@@ -67,7 +66,7 @@ public class ExtendedBlocklet extends Blocklet {
}
public String getPath() {
- return path;
+ return getBlockId();
}
public String getDataMapWriterPath() {
@@ -77,4 +76,30 @@ public class ExtendedBlocklet extends Blocklet {
public void setDataMapWriterPath(String dataMapWriterPath) {
this.dataMapWriterPath = dataMapWriterPath;
}
+
+ public String getDataMapUniqueId() {
+ return dataMapUniqueId;
+ }
+
+ public void setDataMapUniqueId(String dataMapUniqueId) {
+ this.dataMapUniqueId = dataMapUniqueId;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) {
+ return false;
+ }
+
+ ExtendedBlocklet that = (ExtendedBlocklet) o;
+
+ return segmentId != null ? segmentId.equals(that.segmentId) : that.segmentId == null;
+ }
+
+ @Override public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + (segmentId != null ? segmentId.hashCode() : 0);
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
index 266120e..229e5bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
@@ -117,4 +117,12 @@ public class FineGrainBlocklet extends Blocklet implements Serializable {
pages.add(page);
}
}
+
+ @Override public boolean equals(Object o) {
+ return super.equals(o);
+ }
+
+ @Override public int hashCode() {
+ return super.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index b379ae3..90178b1 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -80,8 +80,6 @@ public class BlockletDataMap extends AbstractCoarseGrainDataMap implements Cache
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMap.class.getName());
- public static final String NAME = "clustered.btree.blocklet";
-
private static int KEY_INDEX = 0;
private static int MIN_VALUES_INDEX = 1;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index ee849bd..43df813 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -57,6 +58,11 @@ import org.apache.hadoop.fs.RemoteIterator;
public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
implements BlockletDetailsFetcher, SegmentPropertiesFetcher {
+ private static final String NAME = "clustered.btree.blocklet";
+
+ public static final DataMapSchema DATA_MAP_SCHEMA =
+ new DataMapSchema(NAME, BlockletDataMapFactory.class.getName());
+
private AbsoluteTableIdentifier identifier;
// segmentId -> list of index file
@@ -65,7 +71,7 @@ public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainDataMap> cache;
@Override
- public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
this.identifier = identifier;
cache = CacheProvider.getInstance()
.createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
@@ -188,7 +194,6 @@ public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
new BlockletDataMapDistributable(path.toString());
distributable.setLocations(location);
distributables.add(distributable);
-
}
} catch (IOException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/StartsWithExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/StartsWithExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/StartsWithExpression.java
new file mode 100644
index 0000000..18c7374
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/StartsWithExpression.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class StartsWithExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = -5319109756575539219L;
+
+ public StartsWithExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult exprLeftRes = left.evaluate(value);
+ ExpressionResult exprRightRes = right.evaluate(value);
+ ExpressionResult val1 = exprLeftRes;
+ if (exprLeftRes.isNull() || exprRightRes.isNull()) {
+ exprLeftRes.set(DataTypes.BOOLEAN, false);
+ return exprLeftRes;
+ }
+ if (exprLeftRes.getDataType() != exprRightRes.getDataType()) {
+ if (exprLeftRes.getDataType().getPrecedenceOrder() < exprRightRes.getDataType()
+ .getPrecedenceOrder()) {
+ val1 = exprRightRes;
+ }
+
+ }
+ boolean result = false;
+ DataType dataType = val1.getDataType();
+ if (dataType == DataTypes.STRING) {
+ result = exprLeftRes.getString().startsWith(exprRightRes.getString());
+ } else {
+ throw new FilterUnsupportedException(
+ "DataType: " + val1.getDataType() + " not supported for the filter expression");
+ }
+ val1.set(DataTypes.BOOLEAN, result);
+ return val1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.STARTSWITH;
+ }
+
+ @Override public String getString() {
+ return "StartsWith(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 26b202f..0ccc990 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -46,7 +46,9 @@ import org.apache.carbondata.core.scan.expression.conditional.InExpression;
import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.conditional.StartsWithExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
@@ -370,7 +372,23 @@ public class FilterExpressionProcessor implements FilterProcessor {
case LESSTHAN_EQUALTO:
return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, true, expressionTree,
tableIdentifier, expressionTree);
-
+ case STARTSWITH:
+ assert (expressionTree instanceof StartsWithExpression);
+ currentExpression = (StartsWithExpression) expressionTree;
+ Expression re = currentExpression.getRight();
+ assert (re instanceof LiteralExpression);
+ LiteralExpression literal = (LiteralExpression) re;
+ String value = literal.getLiteralExpValue().toString();
+ Expression left = new GreaterThanEqualToExpression(currentExpression.getLeft(), literal);
+ String maxValueLimit = value.substring(0, value.length() - 1) + (char) (
+ ((int) value.charAt(value.length() - 1)) + 1);
+ Expression right = new LessThanExpression(currentExpression.getLeft(),
+ new LiteralExpression(maxValueLimit, literal.getLiteralExpDataType()));
+ currentExpression = new AndExpression(left, right);
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
+ currentExpression);
case NOT_EQUALS:
case NOT_IN:
return getFilterResolverBasedOnExpressionType(ExpressionType.NOT_EQUALS, false,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
index a3f9199..831acc8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
@@ -39,6 +39,8 @@ public enum ExpressionType {
LITERAL,
RANGE,
FALSE,
- TRUE
-
+ TRUE,
+ STARTSWITH,
+ ENDSWITH,
+ CONTAINSWITH
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index d76158e..820a5a4 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -388,9 +388,8 @@ public class SegmentStatusManager {
LoadMetadataDetails[] listOfLoadFolderDetailsArray = null;
if (!FileFactory.isFileExist(dataLoadLocation, FileFactory.getFileType(dataLoadLocation))) {
- // log error.
- LOG.error("Error message: " + "Load metadata file is not present.");
- invalidLoadTimestamps.add(loadDate);
+ // Table status file is not present, maybe table is empty, ignore this operation
+ LOG.warn("Trying to update table metadata file which is not present.");
return invalidLoadTimestamps;
}
// read existing metadata details in load metadata.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index 4ef74a7..9a67644 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataM
import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
@@ -42,7 +43,7 @@ public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
- @Override public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ @Override public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
this.identifier = identifier;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index f629d40..3bc4547 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -32,10 +32,12 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.DataMapChooser;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
@@ -346,9 +348,6 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
if (null == carbonTable) {
throw new IOException("Missing/Corrupt schema file for table.");
}
- TableDataMap blockletMap =
- DataMapStoreManager.getInstance().getDataMap(identifier, BlockletDataMap.NAME,
- BlockletDataMapFactory.class.getName());
List<Segment> invalidSegments = new ArrayList<>();
List<UpdateVO> invalidTimestampsList = new ArrayList<>();
List<Segment> streamSegments = null;
@@ -379,7 +378,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
.add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId.getSegmentNo()));
}
if (invalidSegments.size() > 0) {
- blockletMap.clear(invalidSegments);
+ DataMapStoreManager.getInstance()
+ .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), invalidSegments);
}
}
ArrayList<Segment> validAndInProgressSegments = new ArrayList<>(segments.getValidSegments());
@@ -407,7 +407,11 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
toBeCleanedSegments.add(segment);
}
}
- blockletMap.clear(toBeCleanedSegments);
+ if (toBeCleanedSegments.size() > 0) {
+ DataMapStoreManager.getInstance()
+ .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()),
+ toBeCleanedSegments);
+ }
// process and resolve the expression
Expression filter = getFilterPredicates(job.getConfiguration());
@@ -743,19 +747,21 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
boolean distributedCG = Boolean.parseBoolean(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT));
- TableDataMap blockletMap =
- DataMapStoreManager.getInstance().chooseDataMap(absoluteTableIdentifier);
+ DataMapExprWrapper dataMapExprWrapper =
+ DataMapChooser.get().choose(getOrCreateCarbonTable(job.getConfiguration()), resolver);
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
- if (distributedCG || blockletMap.getDataMapFactory().getDataMapType() == DataMapType.FG) {
+ if (distributedCG || dataMapExprWrapper.getDataMapType() == DataMapType.FG) {
DistributableDataMapFormat datamapDstr =
- new DistributableDataMapFormat(absoluteTableIdentifier, blockletMap.getDataMapName(),
+ new DistributableDataMapFormat(absoluteTableIdentifier, dataMapExprWrapper,
segmentIds, partitionsToPrune,
BlockletDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
+ // Apply expression on the blocklets.
+ prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets);
} else {
- prunedBlocklets = blockletMap.prune(segmentIds, resolver, partitionsToPrune);
+ prunedBlocklets = dataMapExprWrapper.prune(segmentIds, partitionsToPrune);
}
List<org.apache.carbondata.hadoop.CarbonInputSplit> resultFilterredBlocks = new ArrayList<>();
@@ -921,12 +927,10 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
*/
public BlockMappingVO getBlockRowCount(Job job, AbsoluteTableIdentifier identifier,
List<PartitionSpec> partitions) throws IOException {
- TableDataMap blockletMap = DataMapStoreManager.getInstance()
- .getDataMap(identifier, BlockletDataMap.NAME, BlockletDataMapFactory.class.getName());
+ TableDataMap blockletMap = DataMapStoreManager.getInstance().getDefaultDataMap(identifier);
LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager
.readTableStatusFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()));
- SegmentUpdateStatusManager updateStatusManager =
- new SegmentUpdateStatusManager(identifier, loadMetadataDetails);
+ SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier, loadMetadataDetails);
SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments =
new SegmentStatusManager(identifier).getValidAndInvalidSegments(loadMetadataDetails);
Map<String, Long> blockRowCountMapping = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DataMapJob.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DataMapJob.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DataMapJob.java
index fad2336..64936aa 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DataMapJob.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DataMapJob.java
@@ -29,6 +29,6 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
public interface DataMapJob extends Serializable {
List<ExtendedBlocklet> execute(DistributableDataMapFormat dataMapFormat,
- FilterResolverIntf resolverIntf);
+ FilterResolverIntf filter);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
index 7c1808f..60c88dc 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/DistributableDataMapFormat.java
@@ -22,10 +22,11 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapDistributableWrapper;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -49,7 +50,7 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
private AbsoluteTableIdentifier identifier;
- private String dataMapName;
+ private DataMapExprWrapper dataMapExprWrapper;
private List<Segment> validSegments;
@@ -57,10 +58,11 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
private List<PartitionSpec> partitions;
- public DistributableDataMapFormat(AbsoluteTableIdentifier identifier, String dataMapName,
- List<Segment> validSegments, List<PartitionSpec> partitions, String className) {
+ DistributableDataMapFormat(AbsoluteTableIdentifier identifier,
+ DataMapExprWrapper dataMapExprWrapper, List<Segment> validSegments,
+ List<PartitionSpec> partitions, String className) {
this.identifier = identifier;
- this.dataMapName = dataMapName;
+ this.dataMapExprWrapper = dataMapExprWrapper;
this.validSegments = validSegments;
this.className = className;
this.partitions = partitions;
@@ -85,9 +87,8 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
- TableDataMap dataMap =
- DataMapStoreManager.getInstance().getDataMap(identifier, dataMapName, className);
- List<DataMapDistributable> distributables = dataMap.toDistributable(validSegments);
+ List<DataMapDistributableWrapper> distributables =
+ dataMapExprWrapper.toDistributable(validSegments);
List<InputSplit> inputSplits = new ArrayList<>(distributables.size());
inputSplits.addAll(distributables);
return inputSplits;
@@ -103,13 +104,16 @@ public class DistributableDataMapFormat extends FileInputFormat<Void, ExtendedBl
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException {
- DataMapDistributable distributable = (DataMapDistributable)inputSplit;
+ DataMapDistributableWrapper distributable = (DataMapDistributableWrapper) inputSplit;
TableDataMap dataMap = DataMapStoreManager.getInstance()
- .getDataMap(identifier, distributable.getDataMapName(),
- distributable.getDataMapFactoryClass());
- blockletIterator = dataMap.prune(
- distributable, getFilterExp(taskAttemptContext.getConfiguration()), partitions)
- .iterator();
+ .getDataMap(identifier, distributable.getDistributable().getDataMapSchema());
+ List<ExtendedBlocklet> blocklets = dataMap.prune(
+ distributable.getDistributable(),
+ dataMapExprWrapper.getFilterResolverIntf(distributable.getUniqueId()), partitions);
+ for (ExtendedBlocklet blocklet: blocklets) {
+ blocklet.setDataMapUniqueId(distributable.getUniqueId());
+ }
+ blockletIterator = blocklets.iterator();
}
@Override
[18/54] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad]
Optimization in reading/writing for sort temp row
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 11b3d43..527452a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,15 +31,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -63,21 +62,15 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
* entry count
*/
private int entryCount;
-
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -100,27 +93,21 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
private int numberOfObjectRead;
- private int nullSetWordsLength;
-
- private Comparator<Object[]> comparator;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator<IntermediateSortTempRow> comparator;
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters parameters) {
// set temp file
this.tempFile = tempFile;
- this.dimCnt = parameters.getDimColCount();
- this.complexCnt = parameters.getComplexDimColCount();
- this.measureCnt = parameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-
+ this.tableFieldStat = new TableFieldStat(parameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.executorService = Executors.newFixedThreadPool(1);
- this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
- comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
+ comparator = new IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -169,11 +156,17 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
+ @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+ this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ this.numberOfObjectRead++;
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException("Problems while reading row", e);
+ }
}
}
@@ -207,63 +200,22 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
}
/**
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * get a batch of row, this interface is used in reading compressed sort temp files
+ *
+ * @param expected expected number in a batch
+ * @return a batch of row
+ * @throws IOException if error occurs while reading from stream
*/
- private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
- Object[] row = new Object[dimCnt + measureCnt];
- try {
- int dimCount = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- short aShort = stream.readShort();
- byte[] col = new byte[aShort];
- stream.readFully(col);
- row[dimCount] = col;
- } else {
- int anInt = stream.readInt();
- row[dimCount] = anInt;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimCnt; dimCount++) {
- short aShort = stream.readShort();
- byte[] col = new byte[aShort];
- stream.readFully(col);
- row[dimCount] = col;
- }
-
- long[] words = new long[nullSetWordsLength];
- for (int i = 0; i < words.length; i++) {
- words[i] = stream.readLong();
- }
-
- for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
- if (UnsafeCarbonRowPage.isSet(words, mesCount)) {
- DataType dataType = measureDataTypes[mesCount];
- if (dataType == DataTypes.SHORT) {
- row[dimCount + mesCount] = stream.readShort();
- } else if (dataType == DataTypes.INT) {
- row[dimCount + mesCount] = stream.readInt();
- } else if (dataType == DataTypes.LONG) {
- row[dimCount + mesCount] = stream.readLong();
- } else if (dataType == DataTypes.DOUBLE) {
- row[dimCount + mesCount] = stream.readDouble();
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = stream.readShort();
- byte[] bigDecimalInBytes = new byte[aShort];
- stream.readFully(bigDecimalInBytes);
- row[dimCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- }
- }
- return row;
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException(e);
+ private IntermediateSortTempRow[] readBatchedRowFromStream(int expected)
+ throws IOException {
+ IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
+ for (int i = 0; i < expected; i++) {
+ IntermediateSortTempRow holder
+ = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ holders[i] = holder;
}
+ this.numberOfObjectRead += expected;
+ return holders;
}
/**
@@ -271,7 +223,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @return row
*/
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return this.returnRow;
}
@@ -326,9 +278,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@Override public int hashCode() {
int hash = 0;
- hash += 31 * measureCnt;
- hash += 31 * dimCnt;
- hash += 31 * complexCnt;
+ hash += tableFieldStat.hashCode();
hash += tempFile.hashCode();
return hash;
}
@@ -368,16 +318,12 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * @param numberOfRecords number of records to be read
+ * @return batch of intermediate sort temp row
+ * @throws IOException if error occurs reading records from file
*/
- private Object[][] prefetchRecordsFromFile(int numberOfRecords)
- throws CarbonSortKeyAndGroupByException {
- Object[][] records = new Object[numberOfRecords][];
- for (int i = 0; i < numberOfRecords; i++) {
- records[i] = getRowFromStream();
- }
- return records;
+ private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
+ throws IOException {
+ return readBatchedRowFromStream(numberOfRecords);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index 4bbf61b..22673ff 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -21,25 +21,21 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
import java.util.AbstractQueue;
-import java.util.Arrays;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeSortTempFileChunkHolder;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
@@ -69,22 +65,13 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
private int totalNumberOfRecords;
private SortParameters mergerParameters;
-
+ private TableFieldStat tableFieldStat;
private File[] intermediateFiles;
-
private File outPutFile;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
-
- private long[] nullSetWords;
-
- private ByteBuffer rowData;
+ private SortStepRowHandler sortStepRowHandler;
private Throwable throwable;
@@ -97,16 +84,10 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
- this.dimCnt = mergerParameters.getDimColCount();
- this.complexCnt = mergerParameters.getComplexDimColCount();
- this.measureCnt = mergerParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
- this.nullSetWords = new long[((measureCnt - 1) >> 6) + 1];
- // Take size of 2 MB for each row. I think it is high enough to use
- rowData = ByteBuffer.allocate(2 * 1024 * 1024);
+ this.tableFieldStat = new TableFieldStat(mergerParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
}
@Override public Void call() throws Exception {
@@ -165,13 +146,14 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted record from file
+ * This method will be used to get sorted sort temp row from the sort temp files
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile()
+ throws CarbonSortKeyAndGroupByException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -235,7 +217,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
}
/**
@@ -250,12 +232,12 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] next() throws CarbonSortKeyAndGroupByException {
+ private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -272,82 +254,16 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws CarbonSortKeyAndGroupByException problem while writing
+ * @throws IOException problem while writing
*/
- private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException, IOException {
- int dimCount = 0;
- int size = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- rowData.putShort((short) col.length);
- size += 2;
- rowData.put(col);
- size += col.length;
- } else {
- rowData.putInt((int) row[dimCount]);
- size += 4;
- }
- }
-
- // write complex dimensions here.
- int dimensionSize = dimCnt + complexCnt;
- for (; dimCount < dimensionSize; dimCount++) {
- byte[] col = (byte[]) row[dimCount];
- rowData.putShort((short)col.length);
- size += 2;
- rowData.put(col);
- size += col.length;
- }
- Arrays.fill(nullSetWords, 0);
- int nullSetSize = nullSetWords.length * 8;
- int nullLoc = size;
- size += nullSetSize;
- for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
- Object value = row[mesCount + dimensionSize];
- if (null != value) {
- DataType dataType = measureDataTypes[mesCount];
- if (dataType == DataTypes.SHORT) {
- rowData.putShort(size, (Short) value);
- size += 2;
- } else if (dataType == DataTypes.INT) {
- rowData.putInt(size, (Integer) value);
- size += 4;
- } else if (dataType == DataTypes.LONG) {
- rowData.putLong(size, (Long) value);
- size += 8;
- } else if (dataType == DataTypes.DOUBLE) {
- rowData.putDouble(size, (Double) value);
- size += 8;
- } else if (DataTypes.isDecimal(dataType)) {
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(((BigDecimal) value));
- rowData.putShort(size, (short) bigDecimalInBytes.length);
- size += 2;
- for (int i = 0; i < bigDecimalInBytes.length; i++) {
- rowData.put(size++, bigDecimalInBytes[i]);
- }
- }
- UnsafeCarbonRowPage.set(nullSetWords, mesCount);
- } else {
- UnsafeCarbonRowPage.unset(nullSetWords, mesCount);
- }
- }
- for (int i = 0; i < nullSetWords.length; i++) {
- rowData.putLong(nullLoc, nullSetWords[i]);
- nullLoc += 8;
- }
- byte[] rowBytes = new byte[size];
- rowData.position(0);
- rowData.get(rowBytes);
- stream.write(rowBytes);
- rowData.clear();
+ private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
}
private void finish() throws CarbonSortKeyAndGroupByException {
clear();
try {
CarbonUtil.deleteFiles(intermediateFiles);
- rowData.clear();
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate files");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index ce118d9..64f3c25 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -29,7 +29,8 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.processing.loading.sort.SortStepRowUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder;
@@ -55,7 +56,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
private AbstractQueue<SortTempChunkHolder> recordHolderHeapLocal;
private SortParameters parameters;
- private SortStepRowUtil sortStepRowUtil;
+ private SortStepRowHandler sortStepRowHandler;
/**
* tempFileLocation
*/
@@ -68,7 +69,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
public UnsafeSingleThreadFinalSortFilesMerger(SortParameters parameters,
String[] tempFileLocation) {
this.parameters = parameters;
- this.sortStepRowUtil = new SortStepRowUtil(parameters);
+ this.sortStepRowHandler = new SortStepRowHandler(parameters);
this.tempFileLocation = tempFileLocation;
this.tableName = parameters.getTableName();
}
@@ -108,9 +109,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
LOGGER.info("Started adding first record from each page");
for (final UnsafeCarbonRowPage rowPage : rowPages) {
- SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage,
- parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount(), parameters.getNumberOfSortColumns());
+ SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
// initialize
sortTempFileChunkHolder.readRow();
@@ -121,9 +120,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder =
- new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount());
+ new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn());
// initialize
sortTempFileChunkHolder.readRow();
@@ -142,7 +139,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeapLocal.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
} catch (Exception e) {
LOGGER.error(e);
throw new CarbonDataWriterException(e);
@@ -180,12 +177,14 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted row in 3-parted format.
+ * The row will feed the following writer process step.
*
* @return sorted row
*/
public Object[] next() {
- return sortStepRowUtil.convertRow(getSortedRecordFromFile());
+ IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
+ return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
}
/**
@@ -193,8 +192,8 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
*
* @return sorted record sorted record
*/
- private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 038d34c..7435d73 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -389,7 +389,6 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
noDictionarySortColumnMapping, 0, noDictionarySortColumnMapping.length);
}
sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
-
String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
finalMerger =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index 04efa1f..c06819c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -21,7 +21,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.math.BigDecimal;
import java.util.AbstractQueue;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
@@ -29,11 +28,9 @@ import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class IntermediateFileMerger implements Callable<Void> {
@@ -68,17 +65,12 @@ public class IntermediateFileMerger implements Callable<Void> {
private File[] intermediateFiles;
private File outPutFile;
- private int dimCnt;
- private int noDictDimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
private Throwable throwable;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
/**
* IntermediateFileMerger Constructor
*/
@@ -88,14 +80,10 @@ public class IntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
- this.dimCnt = mergerParameters.getDimColCount();
- this.noDictDimCnt = mergerParameters.getNoDictionaryCount();
- this.complexCnt = mergerParameters.getComplexDimColCount();
- this.measureCnt = mergerParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
+ this.tableFieldStat = new TableFieldStat(mergerParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
}
@Override public Void call() throws Exception {
@@ -154,13 +142,14 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted record from file
+ * This method will be used to get the sorted sort temp row from sort temp file
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile()
+ throws CarbonSortKeyAndGroupByException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -227,7 +216,7 @@ public class IntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
}
/**
@@ -242,12 +231,12 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] next() throws CarbonSortKeyAndGroupByException {
+ private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -264,62 +253,10 @@ public class IntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws CarbonSortKeyAndGroupByException problem while writing
+ * @throws IOException problem while writing
*/
- private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException {
- try {
- int[] mdkArray = (int[]) row[0];
- byte[][] nonDictArray = (byte[][]) row[1];
- int mdkIndex = 0;
- int nonDictKeyIndex = 0;
- // write dictionary and non dictionary dimensions here.
- for (boolean nodictinary : isNoDictionaryDimensionColumn) {
- if (nodictinary) {
- byte[] col = nonDictArray[nonDictKeyIndex++];
- stream.writeShort(col.length);
- stream.write(col);
- } else {
- stream.writeInt(mdkArray[mdkIndex++]);
- }
- }
- // write complex
- for (; nonDictKeyIndex < noDictDimCnt + complexCnt; nonDictKeyIndex++) {
- byte[] col = nonDictArray[nonDictKeyIndex++];
- stream.writeShort(col.length);
- stream.write(col);
- }
- // write measure
- int fieldIndex = 0;
- for (int counter = 0; counter < measureCnt; counter++) {
- if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
- stream.write((byte) 1);
- DataType dataType = measureDataTypes[counter];
- if (dataType == DataTypes.BOOLEAN) {
- stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.SHORT) {
- stream.writeShort((short) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.INT) {
- stream.writeInt((int) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.LONG) {
- stream.writeLong((long) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.DOUBLE) {
- stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (DataTypes.isDecimal(dataType)) {
- byte[] bigDecimalInBytes = DataTypeUtil
- .bigDecimalToByte((BigDecimal) NonDictionaryUtil.getMeasure(fieldIndex, row));
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- } else {
- stream.write((byte) 0);
- }
- fieldIndex++;
- }
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
- }
+ private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
}
private void finish() throws CarbonSortKeyAndGroupByException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
new file mode 100644
index 0000000..9b6d1e8
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+
+/**
+ * This class is used as comparator for comparing intermediate sort temp row
+ */
+public class IntermediateSortTempRowComparator implements Comparator<IntermediateSortTempRow> {
+ /**
+ * isSortColumnNoDictionary whether the sort column is not dictionary or not
+ */
+ private boolean[] isSortColumnNoDictionary;
+
+ /**
+ * @param isSortColumnNoDictionary isSortColumnNoDictionary
+ */
+ public IntermediateSortTempRowComparator(boolean[] isSortColumnNoDictionary) {
+ this.isSortColumnNoDictionary = isSortColumnNoDictionary;
+ }
+
+ /**
+ * Below method will be used to compare two sort temp row
+ */
+ public int compare(IntermediateSortTempRow rowA, IntermediateSortTempRow rowB) {
+ int diff = 0;
+ int dictIndex = 0;
+ int nonDictIndex = 0;
+
+ for (boolean isNoDictionary : isSortColumnNoDictionary) {
+
+ if (isNoDictionary) {
+ byte[] byteArr1 = rowA.getNoDictSortDims()[nonDictIndex];
+ byte[] byteArr2 = rowB.getNoDictSortDims()[nonDictIndex];
+ nonDictIndex++;
+
+ int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
+ if (difference != 0) {
+ return difference;
+ }
+ } else {
+ int dimFieldA = rowA.getDictSortDims()[dictIndex];
+ int dimFieldB = rowB.getDictSortDims()[dictIndex];
+ dictIndex++;
+
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ }
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
index d2579d2..3f94533 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
@@ -40,14 +40,11 @@ public class NewRowComparator implements Comparator<Object[]> {
*/
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
-
int index = 0;
for (boolean isNoDictionary : noDictionarySortColumnMaping) {
-
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[index];
-
byte[] byteArr2 = (byte[]) rowB[index];
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
@@ -57,6 +54,7 @@ public class NewRowComparator implements Comparator<Object[]> {
} else {
int dimFieldA = (int) rowA[index];
int dimFieldB = (int) rowB[index];
+
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
@@ -65,7 +63,6 @@ public class NewRowComparator implements Comparator<Object[]> {
index++;
}
-
return diff;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
index e01b587..7538c92 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
@@ -29,7 +29,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
private int numberOfSortColumns;
/**
- * RowComparatorForNormalDims Constructor
+ * NewRowComparatorForNormalDims Constructor
*
* @param numberOfSortColumns
*/
@@ -46,7 +46,6 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
int diff = 0;
for (int i = 0; i < numberOfSortColumns; i++) {
-
int dimFieldA = (int)rowA[i];
int dimFieldB = (int)rowB[i];
diff = dimFieldA - dimFieldB;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
deleted file mode 100644
index 0ae0b93..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-
-import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-
-public class RowComparator implements Comparator<Object[]> {
- /**
- * noDictionaryCount represent number of no dictionary cols
- */
- private int noDictionaryCount;
-
- /**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
- */
- private boolean[] noDictionarySortColumnMaping;
-
- /**
- * @param noDictionarySortColumnMaping
- * @param noDictionaryCount
- */
- public RowComparator(boolean[] noDictionarySortColumnMaping, int noDictionaryCount) {
- this.noDictionaryCount = noDictionaryCount;
- this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
- }
-
- /**
- * Below method will be used to compare two mdkey
- */
- public int compare(Object[] rowA, Object[] rowB) {
- int diff = 0;
-
- int normalIndex = 0;
- int noDictionaryindex = 0;
-
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
-
- if (isNoDictionary) {
- byte[] byteArr1 = (byte[]) rowA[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-
- ByteBuffer buff1 = ByteBuffer.wrap(byteArr1);
-
- // extract a high card dims from complete byte[].
- NonDictionaryUtil
- .extractSingleHighCardDims(byteArr1, noDictionaryindex, noDictionaryCount, buff1);
-
- byte[] byteArr2 = (byte[]) rowB[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-
- ByteBuffer buff2 = ByteBuffer.wrap(byteArr2);
-
- // extract a high card dims from complete byte[].
- NonDictionaryUtil
- .extractSingleHighCardDims(byteArr2, noDictionaryindex, noDictionaryCount, buff2);
-
- int difference = UnsafeComparer.INSTANCE.compareTo(buff1, buff2);
- if (difference != 0) {
- return difference;
- }
- noDictionaryindex++;
- } else {
- int dimFieldA = NonDictionaryUtil.getDimension(normalIndex, rowA);
- int dimFieldB = NonDictionaryUtil.getDimension(normalIndex, rowB);
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- normalIndex++;
- }
-
- }
-
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
deleted file mode 100644
index 0883ae1..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-
-/**
- * This class is used as comparator for comparing dims which are non high cardinality dims.
- * Here the dims will be in form of int[] (surrogates) so directly comparing the integers.
- */
-public class RowComparatorForNormalDims implements Comparator<Object[]> {
- /**
- * dimension count
- */
- private int numberOfSortColumns;
-
- /**
- * RowComparatorForNormalDims Constructor
- *
- * @param numberOfSortColumns
- */
- public RowComparatorForNormalDims(int numberOfSortColumns) {
- this.numberOfSortColumns = numberOfSortColumns;
- }
-
- /**
- * Below method will be used to compare two surrogate keys
- *
- * @see Comparator#compare(Object, Object)
- */
- public int compare(Object[] rowA, Object[] rowB) {
- int diff = 0;
-
- for (int i = 0; i < numberOfSortColumns; i++) {
-
- int dimFieldA = NonDictionaryUtil.getDimension(i, rowA);
- int dimFieldB = NonDictionaryUtil.getDimension(i, rowB);
-
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
index 88695b9..a4ac0ea 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
@@ -37,6 +37,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -71,12 +73,12 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* tableName
*/
private String tableName;
-
+ private SortParameters sortParameters;
+ private SortStepRowHandler sortStepRowHandler;
/**
* tempFileLocation
*/
private String[] tempFileLocation;
- private SortParameters sortParameters;
private int maxThreadForSorting;
@@ -89,6 +91,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.sortParameters = sortParameters;
+ this.sortStepRowHandler = new SortStepRowHandler(sortParameters);
try {
maxThreadForSorting = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD,
@@ -107,8 +110,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
public void startFinalMerge() throws CarbonDataWriterException {
List<File> filesToMerge = getFilesToMergeSort();
- if (filesToMerge.size() == 0)
- {
+ if (filesToMerge.size() == 0) {
LOGGER.info("No file to merge in final merge stage");
return;
}
@@ -125,11 +127,9 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// get all the merged files
List<File> files = new ArrayList<File>(tempFileLocation.length);
- for (String tempLoc : tempFileLocation)
- {
+ for (String tempLoc : tempFileLocation) {
File[] subFiles = new File(tempLoc).listFiles(fileFilter);
- if (null != subFiles && subFiles.length > 0)
- {
+ if (null != subFiles && subFiles.length > 0) {
files.addAll(Arrays.asList(subFiles));
}
}
@@ -226,13 +226,14 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row from the sort temp files
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
public Object[] next() {
- return getSortedRecordFromFile();
+ IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
+ return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
}
/**
@@ -241,8 +242,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
index 57a19bd..c7efbd9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.sort.sortdata;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.ExecutorService;
@@ -32,12 +32,10 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -69,7 +67,8 @@ public class SortDataRows {
private Semaphore semaphore;
private SortParameters parameters;
-
+ private SortStepRowHandler sortStepRowHandler;
+ private ThreadLocal<ByteBuffer> rowBuffer;
private int sortBufferSize;
private SortIntermediateFileMerger intermediateFileMerger;
@@ -79,7 +78,7 @@ public class SortDataRows {
public SortDataRows(SortParameters parameters,
SortIntermediateFileMerger intermediateFileMerger) {
this.parameters = parameters;
-
+ this.sortStepRowHandler = new SortStepRowHandler(parameters);
this.intermediateFileMerger = intermediateFileMerger;
int batchSize = CarbonProperties.getInstance().getBatchSize();
@@ -87,6 +86,12 @@ public class SortDataRows {
this.sortBufferSize = Math.max(parameters.getSortBufferSize(), batchSize);
// observer of writing file in thread
this.threadStatusObserver = new ThreadStatusObserver();
+ this.rowBuffer = new ThreadLocal<ByteBuffer>() {
+ @Override protected ByteBuffer initialValue() {
+ byte[] backedArray = new byte[2 * 1024 * 1024];
+ return ByteBuffer.wrap(backedArray);
+ }
+ };
}
/**
@@ -130,8 +135,7 @@ public class SortDataRows {
semaphore.acquire();
dataSorterAndWriterExecutorService.execute(new DataSorterAndWriter(recordHolderListLocal));
} catch (InterruptedException e) {
- LOGGER.error(e,
- "exception occurred while trying to acquire a semaphore lock: ");
+ LOGGER.error(e, "exception occurred while trying to acquire a semaphore lock: ");
throw new CarbonSortKeyAndGroupByException(e);
}
// create the new holder Array
@@ -158,7 +162,7 @@ public class SortDataRows {
}
intermediateFileMerger.startMergingIfPossible();
Object[][] recordHolderListLocal = recordHolderList;
- sizeLeft = sortBufferSize - entryCount ;
+ sizeLeft = sortBufferSize - entryCount;
if (sizeLeft > 0) {
System.arraycopy(rowBatch, 0, recordHolderListLocal, entryCount, sizeLeft);
}
@@ -212,7 +216,6 @@ public class SortDataRows {
locationChosen + File.separator + parameters.getTableName() +
System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(recordHolderList, this.entryCount, file);
-
}
startFileBasedMerge();
@@ -220,7 +223,7 @@ public class SortDataRows {
}
/**
- * Below method will be used to write data to file
+ * Below method will be used to write data to sort temp file
*
* @throws CarbonSortKeyAndGroupByException problem while writing
*/
@@ -233,60 +236,9 @@ public class SortDataRows {
parameters.getFileWriteBufferSize(), parameters.getSortTempCompressorName());
// write number of entries to the file
stream.writeInt(entryCountLocal);
- int complexDimColCount = parameters.getComplexDimColCount();
- int dimColCount = parameters.getDimColCount() + complexDimColCount;
- DataType[] type = parameters.getMeasureDataType();
- boolean[] noDictionaryDimnesionMapping = parameters.getNoDictionaryDimnesionColumn();
- Object[] row = null;
for (int i = 0; i < entryCountLocal; i++) {
- // get row from record holder list
- row = recordHolderList[i];
- int dimCount = 0;
- // write dictionary and non dictionary dimensions here.
- for (; dimCount < noDictionaryDimnesionMapping.length; dimCount++) {
- if (noDictionaryDimnesionMapping[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- stream.writeShort(col.length);
- stream.write(col);
- } else {
- stream.writeInt((int)row[dimCount]);
- }
- }
- // write complex dimensions here.
- for (; dimCount < dimColCount; dimCount++) {
- byte[] value = (byte[])row[dimCount];
- stream.writeShort(value.length);
- stream.write(value);
- }
- // as measures are stored in separate array.
- for (int mesCount = 0;
- mesCount < parameters.getMeasureColCount(); mesCount++) {
- Object value = row[mesCount + dimColCount];
- if (null != value) {
- stream.write((byte) 1);
- DataType dataType = type[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- stream.writeBoolean((boolean) value);
- } else if (dataType == DataTypes.SHORT) {
- stream.writeShort((Short) value);
- } else if (dataType == DataTypes.INT) {
- stream.writeInt((Integer) value);
- } else if (dataType == DataTypes.LONG) {
- stream.writeLong((Long) value);
- } else if (dataType == DataTypes.DOUBLE) {
- stream.writeDouble((Double) value);
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal val = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + type[mesCount]);
- }
- } else {
- stream.write((byte) 0);
- }
- }
+ sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToOutputStream(
+ recordHolderList[i], stream, rowBuffer.get());
}
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
@@ -301,7 +253,7 @@ public class SortDataRows {
*
* @throws CarbonSortKeyAndGroupByException
*/
- public void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
+ private void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
CarbonDataProcessorUtil.deleteSortLocationIfExists(parameters.getTempFileLocation());
}
@@ -380,7 +332,8 @@ public class SortDataRows {
// intermediate merging of sort temp files will be triggered
intermediateFileMerger.addFileToMerge(sortTempFile);
LOGGER.info("Time taken to sort and write sort temp file " + sortTempFile + " is: " + (
- System.currentTimeMillis() - startTime));
+ System.currentTimeMillis() - startTime) + ", sort temp file size in MB is "
+ + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
} catch (Throwable e) {
try {
threadStatusObserver.notifyFailed(e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index d726539..7e221a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -21,6 +21,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Comparator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -30,14 +31,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHolder> {
@@ -71,20 +69,13 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int noDictDimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private boolean[] isNoDictionarySortColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -104,7 +95,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
* totalRecordFetch
*/
private int totalRecordFetch;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator<IntermediateSortTempRow> comparator;
/**
* Constructor to initialize
*
@@ -115,16 +108,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
public SortTempFileChunkHolder(File tempFile, SortParameters sortParameters, String tableName) {
// set temp file
this.tempFile = tempFile;
- this.dimCnt = sortParameters.getDimColCount();
- this.noDictDimCnt = sortParameters.getNoDictionaryCount();
- this.complexCnt = sortParameters.getComplexDimColCount();
- this.measureCnt = sortParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = sortParameters.getNoDictionaryDimnesionColumn();
- this.isNoDictionarySortColumn = sortParameters.getNoDictionarySortColumn();
- this.measureDataTypes = sortParameters.getMeasureDataType();
this.readBufferSize = sortParameters.getBufferSize();
this.compressorName = sortParameters.getSortTempCompressorName();
-
+ this.tableFieldStat = new TableFieldStat(sortParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+ this.comparator = new IntermediateSortTempRowComparator(
+ tableFieldStat.getIsSortColNoDictFlags());
this.executorService = Executors
.newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName));
}
@@ -178,7 +167,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+ this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ this.numberOfObjectRead++;
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException("Problem while reading rows", e);
+ }
}
}
@@ -212,86 +206,28 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
/**
- * Reads row from file
+ * Read a batch of row from stream
+ *
* @return Object[]
- * @throws CarbonSortKeyAndGroupByException
+ * @throws IOException if error occurs while reading from stream
*/
- private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
-
- Object[] holder = new Object[3];
- int index = 0;
- int nonDicIndex = 0;
- int[] dim = new int[dimCnt - noDictDimCnt];
- byte[][] nonDicArray = new byte[noDictDimCnt + complexCnt][];
- Object[] measures = new Object[measureCnt];
- try {
- // read dimension values
- for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
- if (isNoDictionaryDimensionColumn[i]) {
- short len = stream.readShort();
- byte[] array = new byte[len];
- stream.readFully(array);
- nonDicArray[nonDicIndex++] = array;
- } else {
- dim[index++] = stream.readInt();
- }
- }
-
- for (int i = 0; i < complexCnt; i++) {
- short len = stream.readShort();
- byte[] array = new byte[len];
- stream.readFully(array);
- nonDicArray[nonDicIndex++] = array;
- }
-
- index = 0;
- // read measure values
- for (int i = 0; i < measureCnt; i++) {
- if (stream.readByte() == 1) {
- DataType dataType = measureDataTypes[i];
- if (dataType == DataTypes.BOOLEAN) {
- measures[index++] = stream.readBoolean();
- } else if (dataType == DataTypes.SHORT) {
- measures[index++] = stream.readShort();
- } else if (dataType == DataTypes.INT) {
- measures[index++] = stream.readInt();
- } else if (dataType == DataTypes.LONG) {
- measures[index++] = stream.readLong();
- } else if (dataType == DataTypes.DOUBLE) {
- measures[index++] = stream.readDouble();
- } else if (DataTypes.isDecimal(dataType)) {
- int len = stream.readInt();
- byte[] buff = new byte[len];
- stream.readFully(buff);
- measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- } else {
- measures[index++] = null;
- }
- }
-
- NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
-
- // increment number if record read
- this.numberOfObjectRead++;
- } catch (IOException e) {
- LOGGER.error("Problme while reading the madkey fom sort temp file");
- throw new CarbonSortKeyAndGroupByException("Problem while reading the sort temp file ", e);
+ private IntermediateSortTempRow[] readBatchedRowFromStream(int expected) throws IOException {
+ IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
+ for (int i = 0; i < expected; i++) {
+ IntermediateSortTempRow holder
+ = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ holders[i] = holder;
}
-
- //return out row
- return holder;
+ this.numberOfObjectRead += expected;
+ return holders;
}
/**
- * below method will be used to get the row
+ * below method will be used to get the sort temp row
*
* @return row
*/
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return this.returnRow;
}
@@ -330,31 +266,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
@Override public int compareTo(SortTempFileChunkHolder other) {
- int diff = 0;
- int index = 0;
- int noDictionaryIndex = 0;
- int[] leftMdkArray = (int[]) returnRow[0];
- int[] rightMdkArray = (int[]) other.returnRow[0];
- byte[][] leftNonDictArray = (byte[][]) returnRow[1];
- byte[][] rightNonDictArray = (byte[][]) other.returnRow[1];
- for (boolean isNoDictionary : isNoDictionarySortColumn) {
- if (isNoDictionary) {
- diff = UnsafeComparer.INSTANCE
- .compareTo(leftNonDictArray[noDictionaryIndex], rightNonDictArray[noDictionaryIndex]);
- if (diff != 0) {
- return diff;
- }
- noDictionaryIndex++;
- } else {
- diff = leftMdkArray[index] - rightMdkArray[index];
- if (diff != 0) {
- return diff;
- }
- index++;
- }
-
- }
- return diff;
+ return comparator.compare(returnRow, other.getRow());
}
@Override public boolean equals(Object obj) {
@@ -372,9 +284,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
@Override public int hashCode() {
int hash = 0;
- hash += 31 * measureCnt;
- hash += 31 * dimCnt;
- hash += 31 * complexCnt;
+ hash += tableFieldStat.hashCode();
hash += tempFile.hashCode();
return hash;
}
@@ -414,16 +324,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * @param numberOfRecords number of records to be read
+ * @return batch of intermediate sort temp row
+ * @throws IOException if error occurs while reading reading records
*/
- private Object[][] prefetchRecordsFromFile(int numberOfRecords)
- throws CarbonSortKeyAndGroupByException {
- Object[][] records = new Object[numberOfRecords][];
- for (int i = 0; i < numberOfRecords; i++) {
- records[i] = getRowFromStream();
- }
- return records;
+ private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
+ throws IOException {
+ return readBatchedRowFromStream(numberOfRecords);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
new file mode 100644
index 0000000..0d1303a
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * This class is used to hold field information for a table during data loading. These information
+ * will be used to convert/construct/destruct row in sort process step. Because complex field is
+ * processed the same as no-dict-no-sort-simple-dimension, so we treat them as the same and use
+ * `no-dict-no-sort-dim` related variable to represent them in this class.
+ */
+public class TableFieldStat implements Serializable {
+ private static final long serialVersionUID = 201712070950L;
+ private int dictSortDimCnt = 0;
+ private int dictNoSortDimCnt = 0;
+ private int noDictSortDimCnt = 0;
+ private int noDictNoSortDimCnt = 0;
+ // whether sort column is of dictionary type or not
+ private boolean[] isSortColNoDictFlags;
+ private int measureCnt;
+ private DataType[] measureDataType;
+
+ // indices for dict & sort dimension columns
+ private int[] dictSortDimIdx;
+ // indices for dict & no-sort dimension columns
+ private int[] dictNoSortDimIdx;
+ // indices for no-dict & sort dimension columns
+ private int[] noDictSortDimIdx;
+ // indices for no-dict & no-sort dimension columns, including complex columns
+ private int[] noDictNoSortDimIdx;
+ // indices for measure columns
+ private int[] measureIdx;
+
+ public TableFieldStat(SortParameters sortParameters) {
+ int noDictDimCnt = sortParameters.getNoDictionaryCount();
+ int complexDimCnt = sortParameters.getComplexDimColCount();
+ int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;
+ this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();
+ int sortColCnt = isSortColNoDictFlags.length;
+ for (boolean flag : isSortColNoDictFlags) {
+ if (flag) {
+ noDictSortDimCnt++;
+ } else {
+ dictSortDimCnt++;
+ }
+ }
+ this.measureCnt = sortParameters.getMeasureColCount();
+ this.measureDataType = sortParameters.getMeasureDataType();
+
+ // be careful that the default value is 0
+ this.dictSortDimIdx = new int[dictSortDimCnt];
+ this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];
+ this.noDictSortDimIdx = new int[noDictSortDimCnt];
+ this.noDictNoSortDimIdx = new int[noDictDimCnt + complexDimCnt - noDictSortDimCnt];
+ this.measureIdx = new int[measureCnt];
+
+ int tmpNoDictSortCnt = 0;
+ int tmpNoDictNoSortCnt = 0;
+ int tmpDictSortCnt = 0;
+ int tmpDictNoSortCnt = 0;
+ boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();
+
+ for (int i = 0; i < isDimNoDictFlags.length; i++) {
+ if (isDimNoDictFlags[i]) {
+ if (i < sortColCnt && isSortColNoDictFlags[i]) {
+ noDictSortDimIdx[tmpNoDictSortCnt++] = i;
+ } else {
+ noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;
+ }
+ } else {
+ if (i < sortColCnt && !isSortColNoDictFlags[i]) {
+ dictSortDimIdx[tmpDictSortCnt++] = i;
+ } else {
+ dictNoSortDimIdx[tmpDictNoSortCnt++] = i;
+ }
+ }
+ }
+ dictNoSortDimCnt = tmpDictNoSortCnt;
+
+ int base = isDimNoDictFlags.length;
+ // adding complex dimension columns
+ for (int i = 0; i < complexDimCnt; i++) {
+ noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = base + i;
+ }
+ noDictNoSortDimCnt = tmpNoDictNoSortCnt;
+
+ base += complexDimCnt;
+ // indices for measure columns
+ for (int i = 0; i < measureCnt; i++) {
+ measureIdx[i] = base + i;
+ }
+ }
+
+ public int getDictSortDimCnt() {
+ return dictSortDimCnt;
+ }
+
+ public int getDictNoSortDimCnt() {
+ return dictNoSortDimCnt;
+ }
+
+ public int getNoDictSortDimCnt() {
+ return noDictSortDimCnt;
+ }
+
+ public int getNoDictNoSortDimCnt() {
+ return noDictNoSortDimCnt;
+ }
+
+ public boolean[] getIsSortColNoDictFlags() {
+ return isSortColNoDictFlags;
+ }
+
+ public int getMeasureCnt() {
+ return measureCnt;
+ }
+
+ public DataType[] getMeasureDataType() {
+ return measureDataType;
+ }
+
+ public int[] getDictSortDimIdx() {
+ return dictSortDimIdx;
+ }
+
+ public int[] getDictNoSortDimIdx() {
+ return dictNoSortDimIdx;
+ }
+
+ public int[] getNoDictSortDimIdx() {
+ return noDictSortDimIdx;
+ }
+
+ public int[] getNoDictNoSortDimIdx() {
+ return noDictNoSortDimIdx;
+ }
+
+ public int[] getMeasureIdx() {
+ return measureIdx;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof TableFieldStat)) return false;
+ TableFieldStat that = (TableFieldStat) o;
+ return dictSortDimCnt == that.dictSortDimCnt
+ && dictNoSortDimCnt == that.dictNoSortDimCnt
+ && noDictSortDimCnt == that.noDictSortDimCnt
+ && noDictNoSortDimCnt == that.noDictNoSortDimCnt
+ && measureCnt == that.measureCnt;
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,
+ noDictNoSortDimCnt, measureCnt);
+ }
+}
\ No newline at end of file
[04/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
index 553f85e..773fbd7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/PartitionSpliterRawResultIterator.java
@@ -17,20 +17,15 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
public class PartitionSpliterRawResultIterator extends CarbonIterator<Object[]> {
- private CarbonIterator<BatchResult> iterator;
- private BatchResult batch;
+ private CarbonIterator<RowBatch> iterator;
+ private RowBatch batch;
private int counter;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(PartitionSpliterRawResultIterator.class.getName());
-
- public PartitionSpliterRawResultIterator(CarbonIterator<BatchResult> iterator) {
+ public PartitionSpliterRawResultIterator(CarbonIterator<RowBatch> iterator) {
this.iterator = iterator;
}
@@ -65,7 +60,7 @@ public class PartitionSpliterRawResultIterator extends CarbonIterator<Object[]>
* @param batch
* @return
*/
- private boolean checkBatchEnd(BatchResult batch) {
+ private boolean checkBatchEnd(RowBatch batch) {
return !(counter < batch.getSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
index 70d0958..1dd1595 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -37,7 +37,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* Iterator of the Batch raw result.
*/
- private CarbonIterator<BatchResult> detailRawQueryResultIterator;
+ private CarbonIterator<RowBatch> detailRawQueryResultIterator;
/**
* Counter to maintain the row counter.
@@ -55,9 +55,9 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
/**
* batch of the result.
*/
- private BatchResult batch;
+ private RowBatch batch;
- public RawResultIterator(CarbonIterator<BatchResult> detailRawQueryResultIterator,
+ public RawResultIterator(CarbonIterator<RowBatch> detailRawQueryResultIterator,
SegmentProperties sourceSegProperties, SegmentProperties destinationSegProperties) {
this.detailRawQueryResultIterator = detailRawQueryResultIterator;
this.sourceSegProperties = sourceSegProperties;
@@ -155,7 +155,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
* @param batch
* @return
*/
- private boolean checkIfBatchIsProcessedCompletely(BatchResult batch) {
+ private boolean checkIfBatchIsProcessedCompletely(RowBatch batch) {
if (counter < batch.getSize()) {
return false;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
index cc9710e..c7cb00d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -35,10 +35,12 @@ public class VectorDetailQueryResultIterator extends AbstractDetailQueryResultIt
super(infos, queryModel, execService);
}
- @Override public Object next() {
+ @Override
+ public Object next() {
throw new UnsupportedOperationException("call processNextBatch instead");
}
+ @Override
public void processNextBatch(CarbonColumnarBatch columnarBatch) {
synchronized (lock) {
updateDataBlockIterator();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
index cfc2f16..973ce0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
@@ -87,7 +87,4 @@ public class CarbonColumnarBatch {
}
}
- public int getRowsFilteredCount() {
- return rowsFiltered;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
index a5f81b9..59117dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
@@ -18,16 +18,16 @@ package org.apache.carbondata.core.scan.result.vector;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> {
public int offset;
public int size;
public CarbonColumnVector vector;
public int vectorOffset;
- public QueryDimension dimension;
- public QueryMeasure measure;
+ public ProjectionDimension dimension;
+ public ProjectionMeasure measure;
public int ordinal;
public DirectDictionaryGenerator directDictionaryGenerator;
public MeasureDataVectorProcessor.MeasureVectorFiller measureVectorFiller;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index db4c982..8902dfb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -29,7 +29,7 @@ public class MeasureDataVectorProcessor {
void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info);
- void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info);
}
@@ -60,7 +60,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -69,13 +69,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putInt(vectorOffset, (int)dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -117,7 +117,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping,
+ public void fillMeasureVector(int[] filteredRowId,
ColumnPage dataChunk, ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -126,13 +126,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -171,7 +171,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -180,13 +180,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putShort(vectorOffset, (short) dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -225,7 +225,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -234,13 +234,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putLong(vectorOffset, dataChunk.getLong(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -279,7 +279,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -288,7 +288,7 @@ public class MeasureDataVectorProcessor {
int precision = info.measure.getMeasure().getPrecision();
BitSet nullBitSet = dataChunk.getNullBits();
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
@@ -330,7 +330,7 @@ public class MeasureDataVectorProcessor {
}
@Override
- public void fillMeasureVectorForFilter(int[] rowMapping, ColumnPage dataChunk,
+ public void fillMeasureVector(int[] filteredRowId, ColumnPage dataChunk,
ColumnVectorInfo info) {
int offset = info.offset;
int len = offset + info.size;
@@ -339,13 +339,13 @@ public class MeasureDataVectorProcessor {
BitSet nullBitSet = dataChunk.getNullBits();
if (nullBitSet.isEmpty()) {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
vector.putDouble(vectorOffset, dataChunk.getDouble(currentRow));
vectorOffset++;
}
} else {
for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
+ int currentRow = filteredRowId[i];
if (nullBitSet.get(currentRow)) {
vector.putNull(vectorOffset);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
deleted file mode 100644
index bf26ca3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.scanner;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Blocklet scanner class to process the block
- */
-public abstract class AbstractBlockletScanner implements BlockletScanner {
-
- /**
- * block execution info
- */
- protected BlockExecutionInfo blockExecutionInfo;
-
- public QueryStatisticsModel queryStatisticsModel;
-
- private AbstractScannedResult emptyResult;
-
- public AbstractBlockletScanner(BlockExecutionInfo tableBlockExecutionInfos) {
- this.blockExecutionInfo = tableBlockExecutionInfos;
- }
-
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- long startTime = System.currentTimeMillis();
- AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- // adding statistics for valid number of pages
- QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
- validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
- validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- // adding statistics for number of pages
- QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
- totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
- totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
- .getDataBlock().blockletId());
- if (!blockExecutionInfo.isPrefetchBlocklet()) {
- readBlocklet(blocksChunkHolder);
- }
- DimensionRawColumnChunk[] dimensionRawColumnChunks =
- blocksChunkHolder.getDimensionRawDataChunk();
- DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][blocksChunkHolder
- .getDataBlock().numberOfPages()];
- MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
- ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][blocksChunkHolder.getDataBlock()
- .numberOfPages()];
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
- scannedResult.setMeasureChunks(columnPages);
- scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
- scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
- if (blockExecutionInfo.isPrefetchBlocklet()) {
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
- }
- }
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- columnPages[i] = measureRawColumnChunks[i].convertToColumnPage();
- }
- }
- }
- int[] numberOfRows = null;
- if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- numberOfRows = dimensionRawColumnChunks[i].getRowCount();
- break;
- }
- }
- } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- numberOfRows = measureRawColumnChunks[i].getRowCount();
- break;
- }
- }
- }
-
- // count(*) case there would not be any dimensions are measures selected.
- if (numberOfRows == null) {
- numberOfRows = new int[blocksChunkHolder.getDataBlock().numberOfPages()];
- for (int i = 0; i < numberOfRows.length; i++) {
- numberOfRows[i] =
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
- int lastPageSize = blocksChunkHolder.getDataBlock().nodeSize()
- % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- ;
- if (lastPageSize > 0) {
- numberOfRows[numberOfRows.length - 1] = lastPageSize;
- }
- }
- scannedResult.setNumberOfRows(numberOfRows);
- if (!blockExecutionInfo.isPrefetchBlocklet()) {
- scannedResult.fillDataChunks();
- }
- // adding statistics for carbon scan time
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime));
- return scannedResult;
- }
-
- @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
- long startTime = System.currentTimeMillis();
- DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock()
- .getDimensionChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
- blocksChunkHolder.setDimensionRawDataChunk(dimensionRawColumnChunks);
- MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getDataBlock()
- .getMeasureChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
- blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
- // adding statistics for carbon read time
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + (System.currentTimeMillis() - startTime));
- }
-
- @Override public AbstractScannedResult createEmptyResult() {
- if (emptyResult == null) {
- emptyResult = new NonFilterQueryScannedResult(blockExecutionInfo);
- emptyResult.setNumberOfRows(new int[0]);
- emptyResult.setIndexes(new int[0][]);
- }
- return emptyResult;
- }
-
- @Override public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException {
- // For non filter it is always true
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
index 0ed0d43..0a41032 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
@@ -18,9 +18,10 @@ package org.apache.carbondata.core.scan.scanner;
import java.io.IOException;
+import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* Interface for processing the block
@@ -30,31 +31,26 @@ public interface BlockletScanner {
/**
* Checks whether this blocklet required to scan or not based on min max of each blocklet.
- * @param blocksChunkHolder
+ * @param dataBlock
* @return
* @throws IOException
*/
- boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException;
+ boolean isScanRequired(DataRefNode dataBlock);
/**
* Below method will used to process the block data and get the scanned result
*
- * @param blocksChunkHolder block chunk which holds the block data
+ * @param rawBlockletColumnChunks block chunk which holds the block data
* @return scannerResult
* result after processing
*/
- AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ BlockletScannedResult scanBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
throws IOException, FilterUnsupportedException;
/**
* Just reads the blocklet from file, does not uncompress it.
- * @param blocksChunkHolder
+ * @param rawBlockletColumnChunks
*/
- void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException;
+ void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException;
- /**
- * In case if there is no filter satisfies.
- * @return AbstractScannedResult
- */
- AbstractScannedResult createEmptyResult();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
new file mode 100644
index 0000000..1c73d63
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.BitSetGroup;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Below class will be used for filter query processing
+ * this class will be first apply the filter then it will read the column page if
+ * required and return the scanned result
+ */
+public class BlockletFilterScanner extends BlockletFullScanner {
+
+ /**
+ * filter executer to evaluate filter condition
+ */
+ private FilterExecuter filterExecuter;
+ /**
+ * this will be used to apply min max
+ * this will be useful for dimension column which is on the right side
+ * as node finder will always give tentative blocks, if column data stored individually
+ * and data is in sorted order then we can check whether filter is in the range of min max or not
+ * if it present then only we can apply filter on complete data.
+ * this will be very useful in case of sparse data when rows are
+ * repeating.
+ */
+ private boolean isMinMaxEnabled;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ private boolean useBitSetPipeLine;
+
+ public BlockletFilterScanner(BlockExecutionInfo blockExecutionInfo,
+ QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo, queryStatisticsModel);
+ // to check whether min max is enabled or not
+ String minMaxEnableValue = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
+ CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
+ if (null != minMaxEnableValue) {
+ isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
+ }
+ // get the filter tree
+ this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
+ this.queryStatisticsModel = queryStatisticsModel;
+
+ String useBitSetPipeLine = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
+ CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
+ if (null != useBitSetPipeLine) {
+ this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
+ }
+ }
+
+ /**
+ * Below method will be used to process the block
+ *
+ * @param rawBlockletColumnChunks block chunk holder which holds the data
+ * @throws FilterUnsupportedException
+ */
+ @Override
+ public BlockletScannedResult scanBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException, FilterUnsupportedException {
+ return executeFilter(rawBlockletColumnChunks);
+ }
+
+ @Override
+ public boolean isScanRequired(DataRefNode dataBlock) {
+ // adding statistics for number of pages
+ QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+ totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+ totalPagesScanned.getCount() + dataBlock.numberOfPages());
+ // apply min max
+ if (isMinMaxEnabled) {
+ BitSet bitSet = null;
+ // check for implicit include filter instance
+ if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+ String blockletId = blockExecutionInfo.getBlockIdString() +
+ CarbonCommonConstants.FILE_SEPARATOR + dataBlock.blockletIndex();
+ bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+ .isFilterValuesPresentInBlockOrBlocklet(
+ dataBlock.getColumnsMaxValue(),
+ dataBlock.getColumnsMinValue(), blockletId);
+ } else {
+ bitSet = this.filterExecuter
+ .isScanRequired(dataBlock.getColumnsMaxValue(),
+ dataBlock.getColumnsMinValue());
+ }
+ return !bitSet.isEmpty();
+ }
+ return true;
+ }
+
+ @Override
+ public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ long startTime = System.currentTimeMillis();
+ this.filterExecuter.readColumnChunks(rawBlockletColumnChunks);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ }
+
+ /**
+ * This method will process the data in below order
+ * 1. first apply min max on the filter tree and check whether any of the filter
+ * is fall on the range of min max, if not then return empty result
+ * 2. If filter falls on min max range then apply filter on actual
+ * data and get the filtered row index
+ * 3. if row index is empty then return the empty result
+ * 4. if row indexes is not empty then read only those blocks(measure or dimension)
+ * which was present in the query but not present in the filter, as while applying filter
+ * some of the blocks where already read and present in chunk holder so not need to
+ * read those blocks again, this is to avoid reading of same blocks which was already read
+ * 5. Set the blocks and filter indexes to result
+ *
+ * @param rawBlockletColumnChunks
+ * @throws FilterUnsupportedException
+ */
+ private BlockletScannedResult executeFilter(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ long startTime = System.currentTimeMillis();
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ // apply filter on actual data, for each page
+ BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(rawBlockletColumnChunks,
+ useBitSetPipeLine);
+ // if filter result is empty then return with empty result
+ if (bitSetGroup.isEmpty()) {
+ CarbonUtil.freeMemory(rawBlockletColumnChunks.getDimensionRawColumnChunks(),
+ rawBlockletColumnChunks.getMeasureRawColumnChunks());
+
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount() + bitSetGroup.getScannedPages());
+ return createEmptyResult();
+ }
+
+ BlockletScannedResult scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
+ rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ // valid scanned blocklet
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ // adding statistics for valid number of pages
+ QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+ validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+ validPages.getCount() + bitSetGroup.getValidPages());
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount() + bitSetGroup.getScannedPages());
+ int[] pageFilteredRowCount = new int[bitSetGroup.getNumberOfPages()];
+ // get the row indexes from bit set for each page
+ int[][] pageFilteredRowId = new int[bitSetGroup.getNumberOfPages()][];
+ int numPages = pageFilteredRowId.length;
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ BitSet bitSet = bitSetGroup.getBitSet(pageId);
+ if (bitSet != null && !bitSet.isEmpty()) {
+ int[] matchedRowId = new int[bitSet.cardinality()];
+ int index = 0;
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ matchedRowId[index++] = i;
+ }
+ pageFilteredRowCount[pageId] = matchedRowId.length;
+ pageFilteredRowId[pageId] = matchedRowId;
+ }
+ }
+
+ long dimensionReadTime = System.currentTimeMillis();
+ dimensionReadTime = System.currentTimeMillis() - dimensionReadTime;
+
+ FileReader fileReader = rawBlockletColumnChunks.getFileReader();
+
+
+ DimensionRawColumnChunk[] dimensionRawColumnChunks =
+ new DimensionRawColumnChunk[blockExecutionInfo.getTotalNumberDimensionToRead()];
+ int numDimensionChunks = dimensionRawColumnChunks.length;
+ // read dimension chunk blocks from file which is not present
+ for (int chunkIndex = 0; chunkIndex < numDimensionChunks; chunkIndex++) {
+ dimensionRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ }
+ int[][] allSelectedDimensionColumnIndexRange =
+ blockExecutionInfo.getAllSelectedDimensionColumnIndexRange();
+ DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) {
+ System.arraycopy(projectionListDimensionChunk, columnIndexRange[0],
+ dimensionRawColumnChunks, columnIndexRange[0],
+ columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+
+ /*
+ * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
+ * then loading them
+ */
+ int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
+ for (int projectionListDimensionIndex : projectionListDimensionIndexes) {
+ if (null == dimensionRawColumnChunks[projectionListDimensionIndex]) {
+ dimensionRawColumnChunks[projectionListDimensionIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ fileReader, projectionListDimensionIndex);
+ }
+ }
+
+ DimensionColumnPage[][] dimensionColumnPages =
+ new DimensionColumnPage[numDimensionChunks][numPages];
+ for (int chunkIndex = 0; chunkIndex < numDimensionChunks; chunkIndex++) {
+ if (dimensionRawColumnChunks[chunkIndex] != null) {
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ dimensionColumnPages[chunkIndex][pageId] =
+ dimensionRawColumnChunks[chunkIndex].decodeColumnPage(pageId);
+ }
+ }
+ }
+
+
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ new MeasureRawColumnChunk[blockExecutionInfo.getTotalNumberOfMeasureToRead()];
+ int numMeasureChunks = measureRawColumnChunks.length;
+
+ // read the measure chunk blocks which is not present
+ for (int chunkIndex = 0; chunkIndex < numMeasureChunks; chunkIndex++) {
+ if (null != rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ measureRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ }
+ }
+
+ int[][] allSelectedMeasureColumnIndexRange =
+ blockExecutionInfo.getAllSelectedMeasureIndexRange();
+ MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) {
+ System.arraycopy(projectionListMeasureChunk, columnIndexRange[0], measureRawColumnChunks,
+ columnIndexRange[0], columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+ /*
+ * in case projection if the projected measure are not loaded in the ColumnPage
+ * then loading them
+ */
+ int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
+ for (int projectionListMeasureIndex : projectionListMeasureIndexes) {
+ if (null == measureRawColumnChunks[projectionListMeasureIndex]) {
+ measureRawColumnChunks[projectionListMeasureIndex] = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(fileReader, projectionListMeasureIndex);
+ }
+ }
+ ColumnPage[][] measureColumnPages = new ColumnPage[numMeasureChunks][numPages];
+ for (int chunkIndex = 0; chunkIndex < numMeasureChunks; chunkIndex++) {
+ if (measureRawColumnChunks[chunkIndex] != null) {
+ for (int pageId = 0; pageId < numPages; pageId++) {
+ measureColumnPages[chunkIndex][pageId] =
+ measureRawColumnChunks[chunkIndex].decodeColumnPage(pageId);
+ }
+ }
+ }
+
+ scannedResult.setDimensionColumnPages(dimensionColumnPages);
+ scannedResult.setPageFilteredRowId(pageFilteredRowId);
+ scannedResult.setMeasureColumnPages(measureColumnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ scannedResult.setPageFilteredRowCount(pageFilteredRowCount);
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime - dimensionReadTime));
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + dimensionReadTime);
+ return scannedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
new file mode 100644
index 0000000..f0211dc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFullScanner.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+
+/**
+ * Blocklet scanner to do full scan of a blocklet,
+ * returning all projection and filter column chunks
+ */
+public class BlockletFullScanner implements BlockletScanner {
+
+ /**
+ * block execution info
+ */
+ protected BlockExecutionInfo blockExecutionInfo;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ private BlockletScannedResult emptyResult;
+
+ public BlockletFullScanner(BlockExecutionInfo tableBlockExecutionInfos,
+ QueryStatisticsModel queryStatisticsModel) {
+ this.blockExecutionInfo = tableBlockExecutionInfos;
+ this.queryStatisticsModel = queryStatisticsModel;
+ }
+
+ @Override
+ public BlockletScannedResult scanBlocklet(
+ RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException, FilterUnsupportedException {
+ long startTime = System.currentTimeMillis();
+ BlockletScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ // adding statistics for valid number of pages
+ QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+ validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+ validPages.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ // adding statistics for number of pages
+ QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
+ totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
+ totalPagesScanned.getCount() + rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
+ rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ readBlocklet(rawBlockletColumnChunks);
+ }
+ DimensionRawColumnChunk[] dimensionRawColumnChunks =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks();
+ DimensionColumnPage[][] dimensionColumnDataChunks =
+ new DimensionColumnPage[dimensionRawColumnChunks.length][rawBlockletColumnChunks
+ .getDataBlock().numberOfPages()];
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks();
+ ColumnPage[][] measureColumnPages =
+ new ColumnPage[measureRawColumnChunks.length][rawBlockletColumnChunks.getDataBlock()
+ .numberOfPages()];
+ scannedResult.setDimensionColumnPages(dimensionColumnDataChunks);
+ scannedResult.setMeasureColumnPages(measureColumnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].decodeAllColumnPages();
+ }
+ }
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ measureColumnPages[i] = measureRawColumnChunks[i].decodeAllColumnPages();
+ }
+ }
+ }
+ int[] numberOfRows = null;
+ if (blockExecutionInfo.getAllSelectedDimensionColumnIndexRange().length > 0) {
+ for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
+ if (dimensionRawColumnChunks[i] != null) {
+ numberOfRows = dimensionRawColumnChunks[i].getRowCount();
+ break;
+ }
+ }
+ } else if (blockExecutionInfo.getAllSelectedMeasureIndexRange().length > 0) {
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (measureRawColumnChunks[i] != null) {
+ numberOfRows = measureRawColumnChunks[i].getRowCount();
+ break;
+ }
+ }
+ }
+
+ // count(*) case there would not be any dimensions are measures selected.
+ if (numberOfRows == null) {
+ numberOfRows = new int[rawBlockletColumnChunks.getDataBlock().numberOfPages()];
+ for (int i = 0; i < numberOfRows.length; i++) {
+ numberOfRows[i] =
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
+ int lastPageSize = rawBlockletColumnChunks.getDataBlock().numRows()
+ % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ ;
+ if (lastPageSize > 0) {
+ numberOfRows[numberOfRows.length - 1] = lastPageSize;
+ }
+ }
+ scannedResult.setPageFilteredRowCount(numberOfRows);
+ if (!blockExecutionInfo.isPrefetchBlocklet()) {
+ scannedResult.fillDataChunks();
+ }
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ return scannedResult;
+ }
+
+ @Override
+ public void readBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException {
+ long startTime = System.currentTimeMillis();
+ DimensionRawColumnChunk[] dimensionRawColumnChunks = rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunks(rawBlockletColumnChunks.getFileReader(),
+ blockExecutionInfo.getAllSelectedDimensionColumnIndexRange());
+ rawBlockletColumnChunks.setDimensionRawColumnChunks(dimensionRawColumnChunks);
+ MeasureRawColumnChunk[] measureRawColumnChunks = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunks(rawBlockletColumnChunks.getFileReader(),
+ blockExecutionInfo.getAllSelectedMeasureIndexRange());
+ rawBlockletColumnChunks.setMeasureRawColumnChunks(measureRawColumnChunks);
+ // adding statistics for carbon read time
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + (System.currentTimeMillis() - startTime));
+ }
+
+ BlockletScannedResult createEmptyResult() {
+ if (emptyResult == null) {
+ emptyResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+ emptyResult.setPageFilteredRowCount(new int[0]);
+ emptyResult.setPageFilteredRowId(new int[0][]);
+ }
+ return emptyResult;
+ }
+
+ @Override public boolean isScanRequired(DataRefNode dataBlock) {
+ // For non filter it is always true
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
deleted file mode 100644
index e77093b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.scanner.impl;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
-import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.BitSetGroup;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-/**
- * Below class will be used for filter query processing
- * this class will be first apply the filter then it will read the block if
- * required and return the scanned result
- */
-public class FilterScanner extends AbstractBlockletScanner {
-
- /**
- * filter tree
- */
- private FilterExecuter filterExecuter;
- /**
- * this will be used to apply min max
- * this will be useful for dimension column which is on the right side
- * as node finder will always give tentative blocks, if column data stored individually
- * and data is in sorted order then we can check whether filter is in the range of min max or not
- * if it present then only we can apply filter on complete data.
- * this will be very useful in case of sparse data when rows are
- * repeating.
- */
- private boolean isMinMaxEnabled;
-
- private QueryStatisticsModel queryStatisticsModel;
-
- private boolean useBitSetPipeLine;
-
- public FilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- // to check whether min max is enabled or not
- String minMaxEnableValue = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
- CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
- if (null != minMaxEnableValue) {
- isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
- }
- // get the filter tree
- this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
- this.queryStatisticsModel = queryStatisticsModel;
-
- String useBitSetPipeLine = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
- CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
- if (null != useBitSetPipeLine) {
- this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
- }
- }
-
- /**
- * Below method will be used to process the block
- *
- * @param blocksChunkHolder block chunk holder which holds the data
- * @throws FilterUnsupportedException
- */
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- return fillScannedResult(blocksChunkHolder);
- }
-
- @Override public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException {
- // adding statistics for number of pages
- QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
- totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
- totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
- // apply min max
- if (isMinMaxEnabled) {
- BitSet bitSet = null;
- // check for implicit include filter instance
- if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
- String blockletId = blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
- + blocksChunkHolder.getDataBlock().blockletId();
- bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
- .isFilterValuesPresentInBlockOrBlocklet(
- blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue(), blockletId);
- } else {
- bitSet = this.filterExecuter
- .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue());
- }
- if (bitSet.isEmpty()) {
- CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
- blocksChunkHolder.getMeasureRawDataChunk());
- return false;
- }
- }
- return true;
- }
-
- @Override public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
- long startTime = System.currentTimeMillis();
- this.filterExecuter.readBlocks(blocksChunkHolder);
- // adding statistics for carbon read time
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + (System.currentTimeMillis() - startTime));
- }
-
- /**
- * This method will process the data in below order
- * 1. first apply min max on the filter tree and check whether any of the filter
- * is fall on the range of min max, if not then return empty result
- * 2. If filter falls on min max range then apply filter on actual
- * data and get the filtered row index
- * 3. if row index is empty then return the empty result
- * 4. if row indexes is not empty then read only those blocks(measure or dimension)
- * which was present in the query but not present in the filter, as while applying filter
- * some of the blocks where already read and present in chunk holder so not need to
- * read those blocks again, this is to avoid reading of same blocks which was already read
- * 5. Set the blocks and filter indexes to result
- *
- * @param blocksChunkHolder
- * @throws FilterUnsupportedException
- */
- private AbstractScannedResult fillScannedResult(BlocksChunkHolder blocksChunkHolder)
- throws FilterUnsupportedException, IOException {
- long startTime = System.currentTimeMillis();
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
- // apply filter on actual data
- BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
- // if indexes is empty then return with empty result
- if (bitSetGroup.isEmpty()) {
- CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
- blocksChunkHolder.getMeasureRawDataChunk());
-
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime));
-
- QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.PAGE_SCANNED);
- scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
- scannedPages.getCount() + bitSetGroup.getScannedPages());
- return createEmptyResult();
- }
-
- AbstractScannedResult scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
- .getDataBlock().blockletId());
- // valid scanned blocklet
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- // adding statistics for valid number of pages
- QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
- validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
- validPages.getCount() + bitSetGroup.getValidPages());
- QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.PAGE_SCANNED);
- scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
- scannedPages.getCount() + bitSetGroup.getScannedPages());
- int[] rowCount = new int[bitSetGroup.getNumberOfPages()];
- // get the row indexes from bot set
- int[][] indexesGroup = new int[bitSetGroup.getNumberOfPages()][];
- for (int k = 0; k < indexesGroup.length; k++) {
- BitSet bitSet = bitSetGroup.getBitSet(k);
- if (bitSet != null && !bitSet.isEmpty()) {
- int[] indexes = new int[bitSet.cardinality()];
- int index = 0;
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- indexes[index++] = i;
- }
- rowCount[k] = indexes.length;
- indexesGroup[k] = indexes;
- }
- }
- FileHolder fileReader = blocksChunkHolder.getFileReader();
- int[][] allSelectedDimensionBlocksIndexes =
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes();
- long dimensionReadTime = System.currentTimeMillis();
- DimensionRawColumnChunk[] projectionListDimensionChunk = blocksChunkHolder.getDataBlock()
- .getDimensionChunks(fileReader, allSelectedDimensionBlocksIndexes);
- dimensionReadTime = System.currentTimeMillis() - dimensionReadTime;
-
- DimensionRawColumnChunk[] dimensionRawColumnChunks =
- new DimensionRawColumnChunk[blockExecutionInfo.getTotalNumberDimensionBlock()];
- // read dimension chunk blocks from file which is not present
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (null != blocksChunkHolder.getDimensionRawDataChunk()[i]) {
- dimensionRawColumnChunks[i] = blocksChunkHolder.getDimensionRawDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
- for (int j = allSelectedDimensionBlocksIndexes[i][0];
- j <= allSelectedDimensionBlocksIndexes[i][1]; j++) {
- dimensionRawColumnChunks[j] = projectionListDimensionChunk[j];
- }
- }
- long dimensionReadTime1 = System.currentTimeMillis();
- /**
- * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
- * then loading them
- */
- int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
- int projectionListDimensionIndexesLength = projectionListDimensionIndexes.length;
- for (int i = 0; i < projectionListDimensionIndexesLength; i++) {
- if (null == dimensionRawColumnChunks[projectionListDimensionIndexes[i]]) {
- dimensionRawColumnChunks[projectionListDimensionIndexes[i]] =
- blocksChunkHolder.getDataBlock()
- .getDimensionChunk(fileReader, projectionListDimensionIndexes[i]);
- }
- }
- dimensionReadTime += (System.currentTimeMillis() - dimensionReadTime1);
- dimensionReadTime1 = System.currentTimeMillis();
- MeasureRawColumnChunk[] measureRawColumnChunks =
- new MeasureRawColumnChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
- int[][] allSelectedMeasureBlocksIndexes =
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes();
- MeasureRawColumnChunk[] projectionListMeasureChunk = blocksChunkHolder.getDataBlock()
- .getMeasureChunks(fileReader, allSelectedMeasureBlocksIndexes);
- dimensionReadTime += System.currentTimeMillis() - dimensionReadTime1;
- // read the measure chunk blocks which is not present
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (null != blocksChunkHolder.getMeasureRawDataChunk()[i]) {
- measureRawColumnChunks[i] = blocksChunkHolder.getMeasureRawDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
- for (int j = allSelectedMeasureBlocksIndexes[i][0];
- j <= allSelectedMeasureBlocksIndexes[i][1]; j++) {
- measureRawColumnChunks[j] = projectionListMeasureChunk[j];
- }
- }
- dimensionReadTime1 = System.currentTimeMillis();
- /**
- * in case projection if the projected measure are not loaded in the ColumnPage
- * then loading them
- */
- int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
- int projectionListMeasureIndexesLength = projectionListMeasureIndexes.length;
- for (int i = 0; i < projectionListMeasureIndexesLength; i++) {
- if (null == measureRawColumnChunks[projectionListMeasureIndexes[i]]) {
- measureRawColumnChunks[projectionListMeasureIndexes[i]] = blocksChunkHolder.getDataBlock()
- .getMeasureChunk(fileReader, projectionListMeasureIndexes[i]);
- }
- }
- dimensionReadTime += System.currentTimeMillis() - dimensionReadTime1;
- DimensionColumnDataChunk[][] dimensionColumnDataChunks =
- new DimensionColumnDataChunk[dimensionRawColumnChunks.length][indexesGroup.length];
- ColumnPage[][] columnPages =
- new ColumnPage[measureRawColumnChunks.length][indexesGroup.length];
- for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
- if (dimensionRawColumnChunks[i] != null) {
- for (int j = 0; j < indexesGroup.length; j++) {
- dimensionColumnDataChunks[i][j] = dimensionRawColumnChunks[i].convertToDimColDataChunk(j);
- }
- }
- }
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (measureRawColumnChunks[i] != null) {
- for (int j = 0; j < indexesGroup.length; j++) {
- columnPages[i][j] = measureRawColumnChunks[i].convertToColumnPage(j);
- }
- }
- }
- scannedResult.setDimensionChunks(dimensionColumnDataChunks);
- scannedResult.setIndexes(indexesGroup);
- scannedResult.setMeasureChunks(columnPages);
- scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
- scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
- scannedResult.setNumberOfRows(rowCount);
- // adding statistics for carbon scan time
- QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
- scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
- scanTime.getCount() + (System.currentTimeMillis() - startTime - dimensionReadTime));
- QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
- readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
- readTime.getCount() + dimensionReadTime);
- return scannedResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
deleted file mode 100644
index 1373ed5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.scanner.impl;
-
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Non filter processor which will be used for non filter query
- * In case of non filter query we just need to read all the blocks requested in the
- * query and pass it to scanned result
- */
-public class NonFilterScanner extends AbstractBlockletScanner {
-
- public NonFilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- super.queryStatisticsModel = queryStatisticsModel;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
index 2f981b5..6faae03 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
@@ -30,22 +30,17 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
* to store key which is generated using
* key generator
*/
- protected byte[] dictionaryKey;
+ private byte[] dictionaryKey;
/**
* to store no dictionary column data
*/
- protected byte[][] complexTypesKeys;
+ private byte[][] complexTypesKeys;
/**
* to store no dictionary column data
*/
- protected byte[][] noDictionaryKeys;
-
- /**
- * contains value of implicit columns in byte array format
- */
- protected byte[] implicitColumnByteArray;
+ private byte[][] noDictionaryKeys;
public ByteArrayWrapper() {
}
@@ -91,16 +86,6 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
}
/**
- * to get the no dictionary column data
- *
- * @param index of the no dictionary key
- * @return no dictionary key for the index
- */
- public byte[] getComplexTypeByIndex(int index) {
- return this.complexTypesKeys[index];
- }
-
- /**
* to generate the hash code
*/
@Override public int hashCode() {
@@ -201,30 +186,10 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper>, Serializa
}
/**
- * @return the complexTypesKeys
- */
- public byte[][] getComplexTypesKeys() {
- return complexTypesKeys;
- }
-
- /**
* @param complexTypesKeys the complexTypesKeys to set
*/
public void setComplexTypesKeys(byte[][] complexTypesKeys) {
this.complexTypesKeys = complexTypesKeys;
}
- /**
- * @return
- */
- public byte[] getImplicitColumnByteArray() {
- return implicitColumnByteArray;
- }
-
- /**
- * @param implicitColumnByteArray
- */
- public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
- this.implicitColumnByteArray = implicitColumnByteArray;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
deleted file mode 100644
index 8a37d01..0000000
--- a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.stats;
-
-import java.io.Serializable;
-
-public class PartitionStatistic implements Serializable {
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
index 55f0882..ed60d37 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsModel.java
@@ -37,7 +37,4 @@ public class QueryStatisticsModel {
return statisticsTypeAndObjMap;
}
- public void setStatisticsTypeAndObjMap(Map<String, QueryStatistic> statisticsTypeAndObjMap) {
- this.statisticsTypeAndObjMap = statisticsTypeAndObjMap;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index b6a9e36..d6671b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -345,15 +345,6 @@ public class LoadMetadataDetails implements Serializable {
}
/**
- * To get isDeleted property.
- *
- * @return isDeleted
- */
- public String getIsDeleted() {
- return isDeleted;
- }
-
- /**
* To set isDeleted property.
*
* @param isDeleted
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index ab849ce..49d494a 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -678,22 +678,6 @@ public class SegmentStatusManager {
return "";
}
- /**
- * getting the task numbers present in the segment.
- * @param segmentId
- * @return
- */
- public List<String> getUpdatedTasksDetailsForSegment(String segmentId, SegmentUpdateStatusManager
- updateStatusManager) {
- List<String> taskList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- List<String> list = updateStatusManager.getUpdateDeltaFiles(segmentId);
- for (String eachFileName : list) {
- taskList.add(CarbonTablePath.DataFileUtil.getTaskNo(eachFileName));
- }
- return taskList;
- }
-
-
public static class ValidAndInvalidSegmentsInfo {
private final List<Segment> listOfValidSegments;
private final List<Segment> listOfValidUpdatedSegments;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 64c13da..92af9a1 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -48,7 +48,6 @@ import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.TupleIdEnum;
import org.apache.carbondata.core.mutate.UpdateVO;
-import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -159,14 +158,6 @@ public class SegmentUpdateStatusManager {
}
/**
- *
- * @param loadMetadataDetails
- */
- public void setLoadMetadataDetails(LoadMetadataDetails[] loadMetadataDetails) {
- this.segmentDetails = loadMetadataDetails;
- }
-
- /**
* Returns the UpdateStatus Details.
* @return
*/
@@ -193,18 +184,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * Returns all delete delta files of specified block
- *
- * @param tupleId
- * @return
- * @throws Exception
- */
- public List<String> getDeleteDeltaFiles(String tupleId) throws Exception {
- return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
- }
-
-
- /**
* Returns all update delta files of specified Segment.
*
* @param segmentId
@@ -267,20 +246,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * Returns all deleted records of specified block
- *
- * @param tupleId
- * @return
- * @throws Exception
- */
- public Map<Integer, Integer[]> getDeleteDeltaDataFromAllFiles(String tupleId) throws Exception {
- List<String> deltaFiles = getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
- CarbonDeleteFilesDataReader dataReader = new CarbonDeleteFilesDataReader();
- String blockletId = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCKLET_ID);
- return dataReader.getDeleteDataFromAllFiles(deltaFiles, blockletId);
- }
-
- /**
* Below method will be used to get all the delete delta files based on block name
*
* @param blockFilePath actual block filePath
@@ -796,41 +761,6 @@ public class SegmentUpdateStatusManager {
}
/**
- * compares passed time stamp with status file delete timestamp and
- * returns latest timestamp from status file if both are not equal
- * returns null otherwise
- *
- * @param completeBlockName
- * @param timestamp
- * @return
- */
- public String getTimestampForRefreshCache(String completeBlockName, String timestamp) {
- long cacheTimestamp = 0;
- if (null != timestamp) {
- cacheTimestamp = CarbonUpdateUtil.getTimeStampAsLong(timestamp);
- }
- String blockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getBlockName(
- CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
- String segmentId =
- CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
- SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
- readLoadMetadata();
- for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
- if (segmentId.equalsIgnoreCase(block.getSegmentName()) &&
- block.getBlockName().equalsIgnoreCase(blockName) &&
- !CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
- long deleteTimestampFromStatusFile = block.getDeleteDeltaEndTimeAsLong();
- if (Long.compare(deleteTimestampFromStatusFile, cacheTimestamp) == 0) {
- return null;
- } else {
- return block.getDeleteDeltaEndTimestamp();
- }
- }
- }
- return null;
- }
-
- /**
* This method closes the streams
*
* @param streams - streams to close.
@@ -849,85 +779,7 @@ public class SegmentUpdateStatusManager {
}
}
}
- /**
- * Get the invalid tasks in that segment.
- * @param segmentId
- * @return
- */
- public List<String> getInvalidBlockList(String segmentId) {
-
- // get the original fact file timestamp from the table status file.
- List<String> listOfInvalidBlocks = new ArrayList<String>();
- SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getTablePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
- LoadMetadataDetails[] segmentDetails =
- ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
- long timestampOfOriginalFacts = 0;
-
- String startTimestampOfUpdate = "" ;
- String endTimestampOfUpdate = "";
-
- for (LoadMetadataDetails segment : segmentDetails) {
- // find matching segment and return timestamp.
- if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
- timestampOfOriginalFacts = segment.getLoadStartTime();
- startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
- endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
- }
- }
-
- if (startTimestampOfUpdate.isEmpty()) {
- return listOfInvalidBlocks;
-
- }
-
- // now after getting the original fact timestamp, what ever is remaining
- // files need to cross check it with table status file.
- // filter out the fact files.
-
- String segmentPath = carbonTablePath.getCarbonDataDirectoryPath(segmentId);
- CarbonFile segDir =
- FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-
- final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
- final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
- final Long timeStampOriginalFactFinal =
- timestampOfOriginalFacts;
-
- CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile pathName) {
- String fileName = pathName.getName();
- if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
- String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
- long timestamp = Long.parseLong(firstPart
- .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- firstPart.length()));
- if (Long.compare(timestamp, endTimeStampFinal) <= 0
- && Long.compare(timestamp, startTimeStampFinal) >= 0) {
- return false;
- }
- if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
- return false;
- }
- // take the rest of files as they are invalid.
- return true;
- }
- return false;
- }
- });
-
- // gather the task numbers.
- for (CarbonFile updateFiles : files) {
- listOfInvalidBlocks.add(updateFiles.getName());
- }
-
- return listOfInvalidBlocks;
- }
/**
* Returns the invalid timestamp range of a segment.
* @param segmentId
@@ -953,12 +805,11 @@ public class SegmentUpdateStatusManager {
}
/**
*
- * @param segmentId
* @param block
* @param needCompleteList
* @return
*/
- public CarbonFile[] getDeleteDeltaInvalidFilesList(final String segmentId,
+ public CarbonFile[] getDeleteDeltaInvalidFilesList(
final SegmentUpdateDetails block, final boolean needCompleteList,
CarbonFile[] allSegmentFiles, boolean isAbortedFile) {
@@ -1004,12 +855,11 @@ public class SegmentUpdateStatusManager {
/**
*
- * @param blockName
* @param allSegmentFiles
* @return
*/
- public CarbonFile[] getAllBlockRelatedFiles(String blockName, CarbonFile[] allSegmentFiles,
- String actualBlockName) {
+ public CarbonFile[] getAllBlockRelatedFiles(CarbonFile[] allSegmentFiles,
+ String actualBlockName) {
List<CarbonFile> files = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (CarbonFile eachFile : allSegmentFiles) {
[15/54] [abbrv] carbondata git commit: Support generating assembling
JAR for store-sdk module
Posted by ra...@apache.org.
Support generating assembling JAR for store-sdk module
Support generating assembling JAR for store-sdk module and remove junit dependency
This closes #1976
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/05396623
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/05396623
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/05396623
Branch: refs/heads/master
Commit: 053966237e8061a9d13131c4abd9a52567ed8642
Parents: 9a423c2
Author: Jacky Li <ja...@qq.com>
Authored: Tue Feb 13 09:12:09 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
common/pom.xml | 2 +
core/pom.xml | 2 +
hadoop/pom.xml | 1 +
integration/presto/pom.xml | 3 +-
integration/spark-common-cluster-test/pom.xml | 2 +-
integration/spark-common-test/pom.xml | 3 +-
integration/spark-common/pom.xml | 2 +-
integration/spark2/pom.xml | 2 +-
pom.xml | 5 +++
processing/pom.xml | 1 +
store/sdk/pom.xml | 50 +++++++++++++++++++++-
streaming/pom.xml | 1 -
12 files changed, 66 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 5550129..433d575 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,10 +42,12 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 92c9607..824de0d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,10 +70,12 @@
<dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop/pom.xml b/hadoop/pom.xml
index 2aaac99..c3964c5 100644
--- a/hadoop/pom.xml
+++ b/hadoop/pom.xml
@@ -42,6 +42,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/integration/presto/pom.xml
----------------------------------------------------------------------
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index 17f5d41..c3c7c64 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -193,7 +193,7 @@
</exclusion>
<exclusion>
<groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.11</artifactId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
@@ -330,7 +330,6 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/integration/spark-common-cluster-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/pom.xml b/integration/spark-common-cluster-test/pom.xml
index fd907a3..028da11 100644
--- a/integration/spark-common-cluster-test/pom.xml
+++ b/integration/spark-common-cluster-test/pom.xml
@@ -49,11 +49,11 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index 67a2317..d1c04ae 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -106,16 +106,17 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jmockit</groupId>
<artifactId>jmockit</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 295d62b..16f327d 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -58,11 +58,11 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/integration/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 73fc3eb..13b3d8d 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -56,11 +56,11 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 520d770..0891972 100644
--- a/pom.xml
+++ b/pom.xml
@@ -262,6 +262,11 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <version>2.2.1</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/processing/pom.xml
----------------------------------------------------------------------
diff --git a/processing/pom.xml b/processing/pom.xml
index 2427039..dfabaa2 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -68,6 +68,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index 51d2cf9..54fba55 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -32,7 +32,6 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -125,6 +124,55 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-install-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <outputFile>carbondata-sdk.jar</outputFile>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>org/datanucleus/**</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/vfs-providers.xml</exclude>
+ <exclude>io/netty/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/05396623/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 968aa8b..01affec 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -39,7 +39,6 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.1</version>
<scope>test</scope>
</dependency>
</dependencies>
[33/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix
bugs in tests in windows env
Posted by ra...@apache.org.
[CARBONDATA-1114][Tests] Fix bugs in tests in windows env
Fix bugs in tests that will cause failure under windows env
This closes #1994
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/859d71c1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/859d71c1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/859d71c1
Branch: refs/heads/master
Commit: 859d71c1737b6287c4f1a06f7cd9055d32ff8a99
Parents: d5396b1
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sat Feb 24 21:18:17 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/dev/DataMap.java | 6 -
.../core/datamap/dev/DataMapFactory.java | 2 +-
.../exception/ConcurrentOperationException.java | 16 +-
.../carbondata/core/locks/LocalFileLock.java | 30 ++--
.../core/metadata/PartitionMapFileStore.java | 0
.../statusmanager/SegmentStatusManager.java | 10 +-
.../SegmentUpdateStatusManager.java | 1 -
.../store/impl/DFSFileReaderImplUnitTest.java | 11 +-
.../store/impl/FileFactoryImplUnitTest.java | 28 +++-
.../filesystem/HDFSCarbonFileTest.java | 3 +-
.../filesystem/LocalCarbonFileTest.java | 20 ++-
datamap/examples/pom.xml | 145 +++++++----------
.../datamap/examples/MinMaxDataWriter.java | 1 -
examples/flink/pom.xml | 4 +-
.../carbondata/examples/FlinkExample.scala | 10 +-
.../CarbonStreamSparkStreamingExample.scala | 1 -
.../hadoop/api/CarbonTableInputFormat.java | 5 +-
.../TestInsertAndOtherCommandConcurrent.scala | 2 +-
.../StandardPartitionGlobalSortTestCase.scala | 2 +-
.../exception/ProcessMetaDataException.java | 2 +
.../org/apache/carbondata/api/CarbonStore.scala | 6 +-
.../carbondata/spark/load/CsvRDDHelper.scala | 157 +++++++++++++++++++
.../load/DataLoadProcessBuilderOnSpark.scala | 3 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../carbondata/spark/util/CommonUtil.scala | 2 -
.../command/carbonTableSchemaCommon.scala | 6 +-
.../CarbonAlterTableCompactionCommand.scala | 3 +-
.../management/CarbonCleanFilesCommand.scala | 2 +-
.../CarbonDeleteLoadByIdCommand.scala | 2 +-
.../CarbonDeleteLoadByLoadDateCommand.scala | 2 +-
.../management/CarbonLoadDataCommand.scala | 28 ++--
.../CarbonProjectForDeleteCommand.scala | 2 +-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../command/table/CarbonDropTableCommand.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 3 -
.../BooleanDataTypesInsertTest.scala | 5 +-
.../vectorreader/AddColumnTestCases.scala | 1 +
.../datamap/DataMapWriterListener.java | 3 +-
.../loading/model/CarbonLoadModelBuilder.java | 34 +++-
.../processing/loading/model/LoadOption.java | 15 +-
.../processing/merger/CarbonDataMergerUtil.java | 3 +-
.../util/CarbonDataProcessorUtil.java | 3 +-
.../processing/util/CarbonLoaderUtil.java | 8 +
.../carbondata/lcm/locks/LocalFileLockTest.java | 2 +-
.../loading/csvinput/CSVInputFormatTest.java | 1 +
store/sdk/pom.xml | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java | 8 +-
48 files changed, 400 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index 02db8af..dd5507c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -38,9 +38,6 @@ public interface DataMap<T extends Blocklet> {
/**
* Prune the datamap with filter expression and partition information. It returns the list of
* blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
*/
List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
List<PartitionSpec> partitions);
@@ -48,9 +45,6 @@ public interface DataMap<T extends Blocklet> {
// TODO Move this method to Abstract class
/**
* Validate whether the current segment needs to be fetching the required data
- *
- * @param filterExp
- * @return
*/
boolean isScanRequired(FilterResolverIntf filterExp);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index 50ac279..d8a467f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -21,8 +21,8 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.events.Event;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java b/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
index 7e717ba..918268c 100644
--- a/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
+++ b/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
@@ -17,21 +17,10 @@
package org.apache.carbondata.core.exception;
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
-
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-/**
- * This exception will be thrown when executing concurrent operations which
- * is not supported in carbon.
- *
- * For example, when INSERT OVERWRITE is executing, other operations are not
- * allowed, so this exception will be thrown
- */
-@InterfaceAudience.User
-@InterfaceStability.Stable
-public class ConcurrentOperationException extends Exception {
+public class ConcurrentOperationException extends MalformedCarbonCommandException {
public ConcurrentOperationException(String dbName, String tableName, String command1,
String command2) {
@@ -48,3 +37,4 @@ public class ConcurrentOperationException extends Exception {
}
}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 75ea074..cb80877 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -17,17 +17,20 @@
package org.apache.carbondata.core.locks;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* This class handles the file locking in the local file system.
@@ -40,11 +43,6 @@ public class LocalFileLock extends AbstractCarbonLock {
private String location;
/**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
* channel is the FileChannel of the lock file.
*/
private FileChannel channel;
@@ -104,8 +102,8 @@ public class LocalFileLock extends AbstractCarbonLock {
FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
}
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
+ channel = FileChannel.open(Paths.get(lockFilePath), StandardOpenOption.WRITE,
+ StandardOpenOption.APPEND);
try {
fileLock = channel.tryLock();
} catch (OverlappingFileLockException e) {
@@ -137,11 +135,17 @@ public class LocalFileLock extends AbstractCarbonLock {
} catch (IOException e) {
status = false;
} finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
+ CarbonUtil.closeStreams(channel);
+
+ // deleting the lock file after releasing the lock.
+ if (null != lockFilePath) {
+ CarbonFile lockFile = FileFactory.getCarbonFile(lockFilePath,
+ FileFactory.getFileType(lockFilePath));
+ if (!lockFile.exists() || lockFile.delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ status = false;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index b3b1240..d76158e 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.locks.CarbonLockFactory;
import org.apache.carbondata.core.locks.CarbonLockUtil;
import org.apache.carbondata.core.locks.ICarbonLock;
@@ -837,6 +838,13 @@ public class SegmentStatusManager {
public static void deleteLoadsAndUpdateMetadata(
CarbonTable carbonTable,
boolean isForceDeletion) throws IOException {
+ deleteLoadsAndUpdateMetadata(carbonTable, isForceDeletion, null);
+ }
+
+ public static void deleteLoadsAndUpdateMetadata(
+ CarbonTable carbonTable,
+ boolean isForceDeletion,
+ List<PartitionSpec> partitionSpecs) throws IOException {
if (isLoadDeletionRequired(carbonTable.getMetadataPath())) {
LoadMetadataDetails[] details =
SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
@@ -881,7 +889,7 @@ public class SegmentStatusManager {
CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
if (updationCompletionStatus) {
DeleteLoadFolders.physicalFactAndMeasureMetadataDeletion(
- identifier, carbonTable.getMetadataPath(), isForceDeletion);
+ identifier, carbonTable.getMetadataPath(), isForceDeletion, partitionSpecs);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 39eb262..a21873d 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -44,7 +44,6 @@ import org.apache.carbondata.core.locks.CarbonLockFactory;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.TupleIdEnum;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
index da61a94..30144c1 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileReaderImplUnitTest.java
@@ -38,7 +38,6 @@ import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
public class DFSFileReaderImplUnitTest {
-
private static DFSFileReaderImpl dfsFileHolder;
private static String fileName;
private static String fileNameWithEmptyContent;
@@ -50,10 +49,8 @@ public class DFSFileReaderImplUnitTest {
file = new File("Test.carbondata");
fileWithEmptyContent = new File("TestEXception.carbondata");
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
+ if (file.exists()) {
+ file.delete();
}
if (!fileWithEmptyContent.exists()) try {
fileWithEmptyContent.createNewFile();
@@ -61,10 +58,12 @@ public class DFSFileReaderImplUnitTest {
e.printStackTrace();
}
try {
- FileOutputStream of = new FileOutputStream(file, true);
+ FileOutputStream of = new FileOutputStream(file, false);
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
br.write("Hello World");
br.close();
+ of.flush();
+ of.close();
} catch (Exception e) {
e.getMessage();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
index 65590d6..0e7d1c9 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -43,6 +44,10 @@ public class FileFactoryImplUnitTest {
@AfterClass
public static void tearDown() {
+ cleanUp();
+ }
+
+ private static void cleanUp() {
File file = new File(filePath);
if (file.exists()) {
file.delete();
@@ -83,17 +88,17 @@ public class FileFactoryImplUnitTest {
}
@Test public void testCreateNewFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithDefaultFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL));
}
@Test public void testCreateNewLockFileWithViewFsFileType() throws IOException {
- tearDown();
+ cleanUp();
assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
}
@@ -129,20 +134,29 @@ public class FileFactoryImplUnitTest {
assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
}
- @Test public void testGetDataOutputStreamUsingAppendeForException() {
+ @Test public void testGetDataOutputStreamUsingAppendeForException() throws IOException {
+ DataOutputStream outputStream = null;
try {
- FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
+ outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
} catch (Exception exception) {
assertEquals("Not supported", exception.getMessage());
+ } finally {
+ if (null != outputStream) {
+ outputStream.close();
+ }
}
}
@Test public void getDataOutputStreamForVIEWFSType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void getDataOutputStreamForLocalType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL));
+ DataOutputStream outputStream = FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL);
+ assertNotNull(outputStream);
+ outputStream.close();
}
@Test public void testGetCarbonFile() throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index 4018123..42d4afa 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -58,7 +58,7 @@ public class HDFSCarbonFileTest {
@BeforeClass
static public void setUp() throws IOException {
Configuration config = new Configuration();
-//adding local hadoop configuration
+ // adding local hadoop configuration
config.addResource(new Path("core-site.xml"));
config.addResource(new Path("hdfs-site.xml"));
fileName = "Test.carbondata"; //this path is HDFS path
@@ -75,6 +75,7 @@ public class HDFSCarbonFileTest {
BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
br.write("Hello World");
br.close();
+ os.close();
fs.close();
fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 96ef106..14f9fe2 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -29,6 +29,7 @@ import sun.nio.ch.FileChannelImpl;
import java.io.*;
import java.nio.channels.ReadableByteChannel;
import java.util.Objects;
+import java.util.UUID;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -44,8 +45,8 @@ public class LocalCarbonFileTest {
@BeforeClass
static public void setUp() {
- file = new File("Test.carbondata");
- dir = new File("Testdir.carbondata");
+ file = new File("TestLocalCarbonFile");
+ dir = new File("TestLocalCarbonDir");
if (!file.exists())
try {
file.createNewFile();
@@ -60,6 +61,7 @@ public class LocalCarbonFileTest {
byte[] bytes = "core java api".getBytes();
oFile.write(bytes);
+ oFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
localCarbonFile = new LocalCarbonFile(file);
@@ -121,8 +123,9 @@ public class LocalCarbonFileTest {
@Test
public void testRenameForce() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
- File file1 = new File("Testdb.carbon");
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(localCarbonFile.renameForce(destFile));
+ File file1 = new File(destFile);
if (file1.exists()) {
file1.delete();
}
@@ -131,7 +134,12 @@ public class LocalCarbonFileTest {
@Test
public void testRenameTo() {
localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+ String destFile = "TestRename" + UUID.randomUUID().toString();
+ assertTrue(!localCarbonFile.renameTo(destFile));
+ File file1 = new File(destFile);
+ if (file1.exists()) {
+ file1.delete();
+ }
}
@Test
@@ -463,6 +471,6 @@ public class LocalCarbonFileTest {
localCarbonFile = new LocalCarbonFile("demo.txt");
- assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+ assertEquals(localCarbonFile.renameForce("renameToFile"), true);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
index 0049950..8539a86 100644
--- a/datamap/examples/pom.xml
+++ b/datamap/examples/pom.xml
@@ -15,97 +15,70 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
+ <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
- <artifactId>carbondata-datamap-examples</artifactId>
- <name>Apache CarbonData :: Datamap Examples</name>
+ <artifactId>carbondata-datamap-examples</artifactId>
+ <name>Apache CarbonData :: DataMap Examples</name>
- <properties>
- <dev.path>${basedir}/../../dev</dev.path>
- </properties>
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark2</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_${scala.binary.version}</artifactId>
- </dependency>
- </dependencies>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
- <build>
- <sourceDirectory>src/minmaxdatamap/main/java</sourceDirectory>
- <resources>
- <resource>
- <directory>.</directory>
- <includes>
- <include>CARBON_EXAMPLESLogResource.properties</include>
- </includes>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <version>2.15.2</version>
- <executions>
- <execution>
- <id>compile</id>
- <goals>
- <goal>compile</goal>
- </goals>
- <phase>compile</phase>
- </execution>
- <execution>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
+ <build>
+ <sourceDirectory>src/minmaxdatamap/main/java</sourceDirectory>
+ <resources>
+ <resource>
+ <directory>.</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 5046182..17c8332 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/examples/flink/pom.xml
----------------------------------------------------------------------
diff --git a/examples/flink/pom.xml b/examples/flink/pom.xml
index 75913cf..b783435 100644
--- a/examples/flink/pom.xml
+++ b/examples/flink/pom.xml
@@ -52,12 +52,12 @@
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark</artifactId>
+ <artifactId>carbondata-spark2</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-examples-spark</artifactId>
+ <artifactId>carbondata-examples-spark2</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/examples/flink/src/main/scala/org/apache/carbondata/examples/FlinkExample.scala
----------------------------------------------------------------------
diff --git a/examples/flink/src/main/scala/org/apache/carbondata/examples/FlinkExample.scala b/examples/flink/src/main/scala/org/apache/carbondata/examples/FlinkExample.scala
index 9ce95ae..239a038 100644
--- a/examples/flink/src/main/scala/org/apache/carbondata/examples/FlinkExample.scala
+++ b/examples/flink/src/main/scala/org/apache/carbondata/examples/FlinkExample.scala
@@ -21,8 +21,8 @@ import org.apache.flink.api.java.ExecutionEnvironment
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.Job
-import org.apache.carbondata.examples.util.ExampleUtils
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
+import org.apache.carbondata.hadoop.CarbonProjection
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
// Write carbondata file by spark and read it by flink
// scalastyle:off println
@@ -30,7 +30,7 @@ object FlinkExample {
def main(args: Array[String]): Unit = {
// write carbondata file by spark
- val cc = ExampleUtils.createCarbonContext("FlinkExample")
+ val cc = ExampleUtils.createCarbonSession("FlinkExample")
val path = ExampleUtils.writeSampleCarbonFile(cc, "carbon1")
// read two columns by flink
@@ -38,11 +38,11 @@ object FlinkExample {
projection.addColumn("c1") // column c1
projection.addColumn("c3") // column c3
val conf = new Configuration()
- CarbonInputFormat.setColumnProjection(conf, projection)
+ CarbonTableInputFormat.setColumnProjection(conf, projection)
val env = ExecutionEnvironment.getExecutionEnvironment
val ds = env.readHadoopFile(
- new CarbonInputFormat[Array[Object]],
+ new CarbonTableInputFormat[Array[Object]],
classOf[Void],
classOf[Array[Object]],
path,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStreamSparkStreamingExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStreamSparkStreamingExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStreamSparkStreamingExample.scala
index 63b1c5a..856084b 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStreamSparkStreamingExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonStreamSparkStreamingExample.scala
@@ -20,7 +20,6 @@ package org.apache.carbondata.examples
import java.io.{File, PrintWriter}
import java.net.ServerSocket
-import org.apache.hadoop.conf.Configuration
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.CarbonEnv
import org.apache.spark.sql.CarbonSparkStreamingFactory
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 5cebc12..f629d40 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -33,8 +33,8 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -498,7 +498,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
for (Segment segment : streamSegments) {
- String segmentDir = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
+ String segmentDir = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), segment.getSegmentNo());
FileFactory.FileType fileType = FileFactory.getFileType(segmentDir);
if (FileFactory.isFileExist(segmentDir, fileType)) {
String indexName = CarbonTablePath.getCarbonStreamIndexFileName();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index b39c44c..3f0ca42 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -34,11 +34,11 @@ import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainData
import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMap}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
import org.apache.carbondata.spark.testsuite.datamap.C2DataMapFactory
// This testsuite test insert and insert overwrite with other commands concurrently
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
index b511ee8..6e6be68 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionGlobalSortTestCase.scala
@@ -24,9 +24,9 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class StandardPartitionGlobalSortTestCase extends QueryTest with BeforeAndAfterAll {
var executorService: ExecutorService = _
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ProcessMetaDataException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ProcessMetaDataException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ProcessMetaDataException.java
index 3e06bde..471b645 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ProcessMetaDataException.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ProcessMetaDataException.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.spark.exception;
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException;
+
// This exception will be thrown when processMetaData failed in
// Carbon's RunnableCommand
public class ProcessMetaDataException extends MalformedCarbonCommandException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index b69ec37..bfb1616 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, SegmentFile
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.spark.util.DataLoadingUtil
object CarbonStore {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
@@ -139,9 +138,8 @@ object CarbonStore {
carbonCleanFilesLock =
CarbonLockUtil
.getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
- SegmentStatusManager.deleteLoadsAndUpdateMetadata(carbonTable, true)
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(
- isForceDeletion = true, carbonTable, currentTablePartitions.map(_.asJava).orNull)
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(
+ carbonTable, true, currentTablePartitions.map(_.asJava).orNull)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
currentTablePartitions match {
case Some(partitions) =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala
new file mode 100644
index 0000000..36d8c51
--- /dev/null
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/CsvRDDHelper.scala
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.load
+
+import java.text.SimpleDateFormat
+import java.util.{Date, Locale}
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapred.JobConf
+import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
+import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
+import org.apache.hadoop.mapreduce.task.{JobContextImpl, TaskAttemptContextImpl}
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile}
+import org.apache.spark.sql.util.SparkSQLUtil.sessionState
+
+import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel
+import org.apache.carbondata.spark.rdd.SerializableConfiguration
+import org.apache.carbondata.spark.util.CommonUtil
+
+object CsvRDDHelper {
+
+ private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+
+ /**
+ * createsw a RDD that does reading of multiple CSV files
+ */
+ def csvFileScanRDD(
+ spark: SparkSession,
+ model: CarbonLoadModel,
+ hadoopConf: Configuration
+ ): RDD[InternalRow] = {
+ // 1. partition
+ val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
+ val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
+ val defaultParallelism = spark.sparkContext.defaultParallelism
+ CommonUtil.configureCSVInputFormat(hadoopConf, model)
+ hadoopConf.set(FileInputFormat.INPUT_DIR, model.getFactFilePath)
+ val jobConf = new JobConf(hadoopConf)
+ SparkHadoopUtil.get.addCredentials(jobConf)
+ val jobContext = new JobContextImpl(jobConf, null)
+ val inputFormat = new CSVInputFormat()
+ val rawSplits = inputFormat.getSplits(jobContext).toArray
+ val splitFiles = rawSplits.map { split =>
+ val fileSplit = split.asInstanceOf[FileSplit]
+ PartitionedFile(
+ InternalRow.empty,
+ fileSplit.getPath.toString,
+ fileSplit.getStart,
+ fileSplit.getLength,
+ fileSplit.getLocations)
+ }.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
+ val totalBytes = splitFiles.map(_.length + openCostInBytes).sum
+ val bytesPerCore = totalBytes / defaultParallelism
+
+ val maxSplitBytes = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
+ LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
+ s"open cost is considered as scanning $openCostInBytes bytes.")
+
+ val partitions = new ArrayBuffer[FilePartition]
+ val currentFiles = new ArrayBuffer[PartitionedFile]
+ var currentSize = 0L
+
+ def closePartition(): Unit = {
+ if (currentFiles.nonEmpty) {
+ val newPartition =
+ FilePartition(
+ partitions.size,
+ currentFiles.toArray.toSeq)
+ partitions += newPartition
+ }
+ currentFiles.clear()
+ currentSize = 0
+ }
+
+ splitFiles.foreach { file =>
+ if (currentSize + file.length > maxSplitBytes) {
+ closePartition()
+ }
+ // Add the given file to the current partition.
+ currentSize += file.length + openCostInBytes
+ currentFiles += file
+ }
+ closePartition()
+
+ // 2. read function
+ val serializableConfiguration = new SerializableConfiguration(jobConf)
+ val readFunction = new (PartitionedFile => Iterator[InternalRow]) with Serializable {
+ override def apply(file: PartitionedFile): Iterator[InternalRow] = {
+ new Iterator[InternalRow] {
+ val hadoopConf = serializableConfiguration.value
+ val jobTrackerId: String = {
+ val formatter = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
+ formatter.format(new Date())
+ }
+ val attemptId = new TaskAttemptID(jobTrackerId, 0, TaskType.MAP, 0, 0)
+ val hadoopAttemptContext = new TaskAttemptContextImpl(hadoopConf, attemptId)
+ val inputSplit =
+ new FileSplit(new Path(file.filePath), file.start, file.length, file.locations)
+ var finished = false
+ val inputFormat = new CSVInputFormat()
+ val reader = inputFormat.createRecordReader(inputSplit, hadoopAttemptContext)
+ reader.initialize(inputSplit, hadoopAttemptContext)
+
+ override def hasNext: Boolean = {
+ if (!finished) {
+ if (reader != null) {
+ if (reader.nextKeyValue()) {
+ true
+ } else {
+ finished = true
+ reader.close()
+ false
+ }
+ } else {
+ finished = true
+ false
+ }
+ } else {
+ false
+ }
+ }
+
+ override def next(): InternalRow = {
+ new GenericInternalRow(reader.getCurrentValue.get().asInstanceOf[Array[Any]])
+ }
+ }
+ }
+ }
+ new FileScanRDD(spark, readFunction, partitions)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
index e1bd84b..1062cd7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessBuilderOnSpark.scala
@@ -34,7 +34,6 @@ import org.apache.carbondata.processing.loading.{DataLoadProcessBuilder, Failure
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.sort.sortdata.{NewRowComparator, NewRowComparatorForNormalDims, SortParameters}
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
-import org.apache.carbondata.spark.util.DataLoadingUtil
/**
* Use sortBy operator in spark to load the data
@@ -52,7 +51,7 @@ object DataLoadProcessBuilderOnSpark {
} else {
// input data from files
val columnCount = model.getCsvHeaderColumns.length
- DataLoadingUtil.csvFileScanRDD(sparkSession, model, hadoopConf)
+ CsvRDDHelper.csvFileScanRDD(sparkSession, model, hadoopConf)
.map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 773ea16..59ed8ba 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.execution.command.{DataTypeInfo, UpdateTableModel}
import org.apache.spark.sql.types._
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogService
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
@@ -46,7 +47,6 @@ import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.FailureCauses
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
object CarbonScalaUtil {
def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 9104a32..d3093fb 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -816,8 +816,6 @@ object CommonUtil {
val carbonTable = CarbonMetadata.getInstance
.getCarbonTable(identifier.getCarbonTableIdentifier.getTableUniqueName)
SegmentStatusManager.deleteLoadsAndUpdateMetadata(carbonTable, true)
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(
- isForceDeletion = true, carbonTable, null)
} catch {
case _: Exception =>
LOGGER.warn(s"Error while cleaning table " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 71ce2c6..3c21af3 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -31,19 +31,15 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.metadata.SegmentFileStore.SegmentFile
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema._
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, RelationIdentifier, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.{ColumnSchema, ParentColumnTableRelation}
-import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator
import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.DataTypeUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.loading.FailureCauses
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.merger.CompactionType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index e47c500..2f4aa30 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -21,7 +21,7 @@ import java.io.{File, IOException}
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonEnv, Row, SQLContext, SparkSession}
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.{AlterTableModel, AtomicRunnableCommand, CarbonMergerMapping, CompactionModel}
@@ -47,7 +47,6 @@ import org.apache.carbondata.processing.loading.events.LoadEvents.LoadMetadataEv
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
-import org.apache.carbondata.spark.util.CommonUtil
import org.apache.carbondata.streaming.StreamHandoffRDD
import org.apache.carbondata.streaming.segment.StreamSegment
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index d2adc57..2092028 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -26,10 +26,10 @@ import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.events.{CleanFilesPostEvent, CleanFilesPreEvent, OperationContext, OperationListenerBus}
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
import org.apache.carbondata.spark.util.CommonUtil
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala
index 0861c63..81427a1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByIdCommand.scala
@@ -21,9 +21,9 @@ import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
import org.apache.spark.sql.execution.command.{Checker, DataCommand}
import org.apache.carbondata.api.CarbonStore
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.events.{DeleteSegmentByIdPostEvent, DeleteSegmentByIdPreEvent, OperationContext, OperationListenerBus}
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
case class CarbonDeleteLoadByIdCommand(
loadIds: Seq[String],
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala
index dcbc6ce..1d76bda 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonDeleteLoadByLoadDateCommand.scala
@@ -21,9 +21,9 @@ import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
import org.apache.spark.sql.execution.command.{Checker, DataCommand}
import org.apache.carbondata.api.CarbonStore
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.events.{DeleteSegmentByDatePostEvent, DeleteSegmentByDatePreEvent, OperationContext, OperationListenerBus}
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
case class CarbonDeleteLoadByLoadDateCommand(
databaseNameOp: Option[String],
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 70134a6..eb00ebf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -59,10 +59,9 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.statusmanager.SegmentStatus
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, DataTypeUtil}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.events.exception.PreEventException
@@ -71,18 +70,15 @@ import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.exception.NoRetryException
-import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
-import org.apache.carbondata.processing.util.CarbonLoaderUtil
+import org.apache.carbondata.processing.loading.model.{CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.sort.SortScopeOptions
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionRDD}
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, GlobalDictionaryUtil}
-import org.apache.carbondata.spark.load.DataLoadProcessorStepOnSpark
+import org.apache.carbondata.spark.load.{CsvRDDHelper, DataLoadProcessorStepOnSpark}
import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
case class CarbonLoadDataCommand(
databaseNameOp: Option[String],
@@ -193,12 +189,18 @@ case class CarbonLoadDataCommand(
carbonLoadModel.setAggLoadRequest(
internalOptions.getOrElse(CarbonCommonConstants.IS_INTERNAL_LOAD_CALL, "false").toBoolean)
carbonLoadModel.setSegmentId(internalOptions.getOrElse("mergedSegmentName", ""))
+
+ val javaPartition = mutable.Map[String, String]()
+ partition.foreach { case (k, v) =>
+ if (v.isEmpty) javaPartition(k) = null else javaPartition(k) = v.get
+ }
+
new CarbonLoadModelBuilder(table).build(
options.asJava,
optionsFinal,
carbonLoadModel,
hadoopConf,
- partition,
+ javaPartition.asJava,
dataFrame.isDefined)
// Delete stale segment folders that are not in table status but are physically present in
// the Fact folder
@@ -231,11 +233,7 @@ case class CarbonLoadDataCommand(
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
// Clean up the old invalid segment data before creating a new entry for new load.
- SegmentStatusManager.deleteLoadsAndUpdateMetadata(table, false)
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(
- isForceDeletion = false,
- table,
- currPartitions)
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(table, false, currPartitions)
// add the start entry for the new load in the table status file
if (updateModel.isEmpty && !table.isHivePartitionTable) {
CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(
@@ -679,7 +677,7 @@ case class CarbonLoadDataCommand(
}
}
val columnCount = carbonLoadModel.getCsvHeaderColumns.length
- val rdd = DataLoadingUtil.csvFileScanRDD(
+ val rdd = CsvRDDHelper.csvFileScanRDD(
sparkSession,
model = carbonLoadModel,
hadoopConf).map(DataLoadProcessorStepOnSpark.toStringArrayRow(_, columnCount))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
index f074285..230378b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
@@ -22,8 +22,8 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.carbondata.common.exceptions.ConcurrentOperationException
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
index 5165342..2a92478 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
@@ -25,10 +25,10 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.types.ArrayType
import org.apache.spark.storage.StorageLevel
-import org.apache.carbondata.common.exceptions.ConcurrentOperationException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index cf77e0f..2503fc3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -25,12 +25,12 @@ import org.apache.spark.sql.execution.command.{AlterTableRenameModel, MetadataCo
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog}
import org.apache.spark.util.AlterTableUtil
-import org.apache.carbondata.common.exceptions.ConcurrentOperationException
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
index 8001a93..0298eea 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDropTableCommand.scala
@@ -28,12 +28,12 @@ import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.events._
-import org.apache.carbondata.spark.exception.{ConcurrentOperationException, ProcessMetaDataException}
case class CarbonDropTableCommand(
ifExistsSet: Boolean,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index 61a31a5..2eed988 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -50,11 +50,8 @@ import org.apache.carbondata.hadoop.api.{CarbonOutputCommitter, CarbonTableOutpu
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWriter
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, Util}
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
-import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, Util}
class CarbonFileFormat
extends FileFormat
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
index f8cfa6b..45edd3d 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/booleantype/BooleanDataTypesInsertTest.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
@@ -555,7 +556,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
@@ -923,7 +924,7 @@ class BooleanDataTypesInsertTest extends QueryTest with BeforeAndAfterEach with
sql(
s"""
- | LOAD DATA LOCAL INPATH '${storeLocation}'
+ | LOAD DATA LOCAL INPATH '${FileFactory.getUpdatedFilePath(storeLocation)}'
| INTO TABLE hive_table
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index 995f041..d94570a 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -29,6 +29,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.spark.exception.ProcessMetaDataException
class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 1104229..66f8bc5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -74,7 +74,8 @@ public class DataMapWriterListener {
}
List<String> columns = factory.getMeta().getIndexedColumns();
List<AbstractDataMapWriter> writers = registry.get(columns);
- AbstractDataMapWriter writer = factory.createWriter(new Segment(segmentId, null), dataWritePath);
+ AbstractDataMapWriter writer = factory.createWriter(
+ new Segment(segmentId, null), dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 17e8dbe..29dfa40 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.processing.loading.model;
import java.io.IOException;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -100,6 +102,26 @@ public class CarbonLoadModelBuilder {
Map<String, String> optionsFinal,
CarbonLoadModel carbonLoadModel,
Configuration hadoopConf) throws InvalidLoadOptionException, IOException {
+ build(options, optionsFinal, carbonLoadModel, hadoopConf, new HashMap<String, String>(), false);
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param options Load options from user input
+ * @param optionsFinal Load options that populated with default values for optional options
+ * @param carbonLoadModel The output load model
+ * @param hadoopConf hadoopConf is needed to read CSV header if there 'fileheader' is not set in
+ * user provided load options
+ * @param partitions partition name map to path
+ * @param isDataFrame true if build for load for dataframe
+ */
+ public void build(
+ Map<String, String> options,
+ Map<String, String> optionsFinal,
+ CarbonLoadModel carbonLoadModel,
+ Configuration hadoopConf,
+ Map<String, String> partitions,
+ boolean isDataFrame) throws InvalidLoadOptionException, IOException {
carbonLoadModel.setTableName(table.getTableName());
carbonLoadModel.setDatabaseName(table.getDatabaseName());
carbonLoadModel.setTablePath(table.getTablePath());
@@ -214,8 +236,18 @@ public class CarbonLoadModelBuilder {
carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter));
carbonLoadModel.setCsvHeader(fileHeader);
carbonLoadModel.setColDictFilePath(column_dict);
+
+ List<String> ignoreColumns = new ArrayList<>();
+ if (!isDataFrame) {
+ for (Map.Entry<String, String> partition : partitions.entrySet()) {
+ if (partition.getValue() != null) {
+ ignoreColumns.add(partition.getKey());
+ }
+ }
+ }
+
carbonLoadModel.setCsvHeaderColumns(
- LoadOption.getCsvHeaderColumns(carbonLoadModel, hadoopConf));
+ LoadOption.getCsvHeaderColumns(carbonLoadModel, hadoopConf, ignoreColumns));
int validatedMaxColumns = validateMaxColumns(
carbonLoadModel.getCsvHeaderColumns(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index 5af4859..bac1a94 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.processing.loading.model;
import java.io.IOException;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.carbondata.common.Maps;
@@ -201,6 +203,16 @@ public class LoadOption {
public static String[] getCsvHeaderColumns(
CarbonLoadModel carbonLoadModel,
Configuration hadoopConf) throws IOException {
+ return getCsvHeaderColumns(carbonLoadModel, hadoopConf, new LinkedList<String>());
+ }
+
+ /**
+ * Return CSV header field names, with partition column
+ */
+ public static String[] getCsvHeaderColumns(
+ CarbonLoadModel carbonLoadModel,
+ Configuration hadoopConf,
+ List<String> staticPartitionCols) throws IOException {
String delimiter;
if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter())) {
delimiter = CarbonCommonConstants.COMMA;
@@ -231,7 +243,7 @@ public class LoadOption {
}
if (!CarbonDataProcessorUtil.isHeaderValid(carbonLoadModel.getTableName(), csvColumns,
- carbonLoadModel.getCarbonDataLoadSchema())) {
+ carbonLoadModel.getCarbonDataLoadSchema(), staticPartitionCols)) {
if (csvFile == null) {
LOG.error("CSV header in DDL is not proper."
+ " Column names in schema and CSV header are not the same.");
@@ -249,4 +261,5 @@ public class LoadOption {
}
return csvColumns;
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 1ab803b..438e7db 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -1013,7 +1013,8 @@ public final class CarbonDataMergerUtil {
CarbonFile[] updateDeltaFiles = null;
Set<String> uniqueBlocks = new HashSet<String>();
- String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), seg.getSegmentNo());
+ String segmentPath = CarbonTablePath.getSegmentPath(
+ identifier.getTablePath(), seg.getSegmentNo());
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
CarbonFile[] allSegmentFiles = segDir.listFiles();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index efd715c..ba2b0c2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -392,7 +392,8 @@ public final class CarbonDataProcessorUtil {
*
* @return data directory path
*/
- public static String createCarbonStoreLocation(String databaseName, String tableName, String segmentId) {
+ public static String createCarbonStoreLocation(String databaseName, String tableName,
+ String segmentId) {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
return CarbonTablePath.getSegmentPath(carbonTable.getTablePath(), segmentId);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 38e5698..a948538 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -16,9 +16,13 @@
*/
package org.apache.carbondata.processing.util;
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.nio.charset.Charset;
import java.util.*;
import org.apache.carbondata.common.logging.LogService;
@@ -36,6 +40,9 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.locks.CarbonLockUtil;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -55,6 +62,7 @@ import org.apache.carbondata.processing.merger.NodeMultiBlockRelation;
import static org.apache.carbondata.core.enums.EscapeSequences.*;
+import com.google.gson.Gson;
public final class CarbonLoaderUtil {
[34/54] [abbrv] carbondata git commit: [CARBONDATA-2091][DataLoad]
Support specifying sort column bounds in data loading
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
deleted file mode 100644
index f605b22..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithBucketingImpl.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort.impl;
-
-import java.io.File;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.schema.BucketingInfo;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.processing.loading.DataField;
-import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
-import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
-import org.apache.carbondata.processing.loading.sort.AbstractMergeSorter;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows;
-import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
-import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeSingleThreadFinalSortFilesMerger;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-
-/**
- * It parallely reads data from array of iterates and do merge sort.
- * First it sorts the data and write to temp files. These temp files will be merge sorted to get
- * final merge sort result.
- * This step is specifically for bucketing, it sorts each bucket data separately and write to
- * temp files.
- */
-public class UnsafeParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(
- UnsafeParallelReadMergeSorterWithBucketingImpl.class.getName());
-
- private SortParameters sortParameters;
-
- private BucketingInfo bucketingInfo;
-
- public UnsafeParallelReadMergeSorterWithBucketingImpl(DataField[] inputDataFields,
- BucketingInfo bucketingInfo) {
- this.bucketingInfo = bucketingInfo;
- }
-
- @Override public void initialize(SortParameters sortParameters) {
- this.sortParameters = sortParameters;
- }
-
- @Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators)
- throws CarbonDataLoadingException {
- UnsafeSortDataRows[] sortDataRows = new UnsafeSortDataRows[bucketingInfo.getNumberOfBuckets()];
- UnsafeIntermediateMerger[] intermediateFileMergers =
- new UnsafeIntermediateMerger[sortDataRows.length];
- int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
- inMemoryChunkSizeInMB = inMemoryChunkSizeInMB / bucketingInfo.getNumberOfBuckets();
- if (inMemoryChunkSizeInMB < 5) {
- inMemoryChunkSizeInMB = 5;
- }
- try {
- for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
- SortParameters parameters = sortParameters.getCopy();
- parameters.setPartitionID(i + "");
- setTempLocation(parameters);
- intermediateFileMergers[i] = new UnsafeIntermediateMerger(parameters);
- sortDataRows[i] =
- new UnsafeSortDataRows(parameters, intermediateFileMergers[i], inMemoryChunkSizeInMB);
- sortDataRows[i].initialize();
- }
- } catch (MemoryException e) {
- throw new CarbonDataLoadingException(e);
- }
- ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
- this.threadStatusObserver = new ThreadStatusObserver(executorService);
- final int batchSize = CarbonProperties.getInstance().getBatchSize();
- try {
- for (int i = 0; i < iterators.length; i++) {
- executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, this
- .threadStatusObserver));
- }
- executorService.shutdown();
- executorService.awaitTermination(2, TimeUnit.DAYS);
- processRowToNextStep(sortDataRows, sortParameters);
- } catch (Exception e) {
- checkError();
- throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
- }
- checkError();
- try {
- for (int i = 0; i < intermediateFileMergers.length; i++) {
- intermediateFileMergers[i].finish();
- }
- } catch (Exception e) {
- throw new CarbonDataLoadingException(e);
- }
-
- Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
- for (int i = 0; i < sortDataRows.length; i++) {
- batchIterator[i] = new MergedDataIterator(batchSize, intermediateFileMergers[i]);
- }
-
- return batchIterator;
- }
-
- private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger() {
- String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
- sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
- false, false);
- // Set the data file location
- String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation,
- File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
- return new UnsafeSingleThreadFinalSortFilesMerger(sortParameters, dataFolderLocation);
- }
-
- @Override public void close() {
- }
-
- /**
- * Below method will be used to process data to next step
- */
- private boolean processRowToNextStep(UnsafeSortDataRows[] sortDataRows, SortParameters parameters)
- throws CarbonDataLoadingException {
- if (null == sortDataRows || sortDataRows.length == 0) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
- try {
- for (int i = 0; i < sortDataRows.length; i++) {
- // start sorting
- sortDataRows[i].startSorting();
- }
- // check any more rows are present
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
- return false;
- } catch (Exception e) {
- throw new CarbonDataLoadingException(e);
- }
- }
-
- private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
- .getLocalDataFolderLocation(parameters.getDatabaseName(), parameters.getTableName(),
- parameters.getTaskNo(), parameters.getSegmentId(),
- false, false);
- String[] tmpLoc = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
- CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
- parameters.setTempFileLocation(tmpLoc);
- }
-
- /**
- * This thread iterates the iterator and adds the rows to @{@link UnsafeSortDataRows}
- */
- private static class SortIteratorThread implements Runnable {
-
- private Iterator<CarbonRowBatch> iterator;
-
- private UnsafeSortDataRows[] sortDataRows;
-
- private ThreadStatusObserver threadStatusObserver;
-
- public SortIteratorThread(Iterator<CarbonRowBatch> iterator,
- UnsafeSortDataRows[] sortDataRows, ThreadStatusObserver threadStatusObserver) {
- this.iterator = iterator;
- this.sortDataRows = sortDataRows;
- this.threadStatusObserver = threadStatusObserver;
- }
-
- @Override
- public void run() {
- try {
- while (iterator.hasNext()) {
- CarbonRowBatch batch = iterator.next();
- int i = 0;
- while (batch.hasNext()) {
- CarbonRow row = batch.next();
- if (row != null) {
- UnsafeSortDataRows sortDataRow = sortDataRows[row.bucketNumber];
- synchronized (sortDataRow) {
- sortDataRow.addRow(row.getData());
- }
- }
- }
- }
- } catch (Exception e) {
- LOGGER.error(e);
- this.threadStatusObserver.notifyFailed(e);
- }
- }
-
- }
-
- private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
-
-
- private int batchSize;
-
- private boolean firstRow;
-
- private UnsafeIntermediateMerger intermediateMerger;
-
- public MergedDataIterator(int batchSize,
- UnsafeIntermediateMerger intermediateMerger) {
- this.batchSize = batchSize;
- this.intermediateMerger = intermediateMerger;
- this.firstRow = true;
- }
-
- private UnsafeSingleThreadFinalSortFilesMerger finalMerger;
-
- @Override public boolean hasNext() {
- if (firstRow) {
- firstRow = false;
- finalMerger = getFinalMerger();
- List<UnsafeCarbonRowPage> rowPages = intermediateMerger.getRowPages();
- finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]),
- intermediateMerger.getMergedPages());
- }
- return finalMerger.hasNext();
- }
-
- @Override public CarbonRowBatch next() {
- int counter = 0;
- CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
- while (finalMerger.hasNext() && counter < batchSize) {
- rowBatch.addRow(new CarbonRow(finalMerger.next()));
- counter++;
- }
- return rowBatch;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
new file mode 100644
index 0000000..99d6627
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/UnsafeParallelReadMergeSorterWithColumnRangeImpl.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort.impl;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.schema.ColumnRangeInfo;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
+import org.apache.carbondata.processing.loading.sort.AbstractMergeSorter;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows;
+import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
+import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeSingleThreadFinalSortFilesMerger;
+import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * It parallely reads data from array of iterates and do merge sort.
+ * First it sorts the data and write to temp files. These temp files will be merge sorted to get
+ * final merge sort result.
+ * This step is specifically for the data loading with specifying column value range, such as
+ * bucketing, sort_column_bounds, it sorts each range of data separately and write to temp files.
+ */
+public class UnsafeParallelReadMergeSorterWithColumnRangeImpl extends AbstractMergeSorter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(
+ UnsafeParallelReadMergeSorterWithColumnRangeImpl.class.getName());
+
+ private SortParameters originSortParameters;
+ private UnsafeIntermediateMerger[] intermediateFileMergers;
+ private int inMemoryChunkSizeInMB;
+ private AtomicLong rowCounter;
+ private ColumnRangeInfo columnRangeInfo;
+ /**
+ * counters to collect information about rows processed by each range
+ */
+ private List<AtomicLong> insideRowCounterList;
+
+ public UnsafeParallelReadMergeSorterWithColumnRangeImpl(AtomicLong rowCounter,
+ ColumnRangeInfo columnRangeInfo) {
+ this.rowCounter = rowCounter;
+ this.columnRangeInfo = columnRangeInfo;
+ }
+
+ @Override public void initialize(SortParameters sortParameters) {
+ this.originSortParameters = sortParameters;
+ int totalInMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
+ inMemoryChunkSizeInMB = totalInMemoryChunkSizeInMB / columnRangeInfo.getNumOfRanges();
+ if (inMemoryChunkSizeInMB < 5) {
+ inMemoryChunkSizeInMB = 5;
+ }
+ this.insideRowCounterList = new ArrayList<>(columnRangeInfo.getNumOfRanges());
+ for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
+ insideRowCounterList.add(new AtomicLong(0));
+ }
+ }
+
+ @Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators)
+ throws CarbonDataLoadingException {
+ UnsafeSortDataRows[] sortDataRows = new UnsafeSortDataRows[columnRangeInfo.getNumOfRanges()];
+ intermediateFileMergers = new UnsafeIntermediateMerger[columnRangeInfo.getNumOfRanges()];
+ SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
+ try {
+ for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
+ SortParameters parameters = originSortParameters.getCopy();
+ parameters.setPartitionID(i + "");
+ parameters.setRangeId(i);
+ sortParameterArray[i] = parameters;
+ setTempLocation(parameters);
+ intermediateFileMergers[i] = new UnsafeIntermediateMerger(parameters);
+ sortDataRows[i] =
+ new UnsafeSortDataRows(parameters, intermediateFileMergers[i], inMemoryChunkSizeInMB);
+ sortDataRows[i].initialize();
+ }
+ } catch (MemoryException e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
+ this.threadStatusObserver = new ThreadStatusObserver(executorService);
+ final int batchSize = CarbonProperties.getInstance().getBatchSize();
+ try {
+ for (int i = 0; i < iterators.length; i++) {
+ executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter,
+ this.insideRowCounterList, this.threadStatusObserver));
+ }
+ executorService.shutdown();
+ executorService.awaitTermination(2, TimeUnit.DAYS);
+ processRowToNextStep(sortDataRows, originSortParameters);
+ } catch (Exception e) {
+ checkError();
+ throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
+ }
+ checkError();
+ try {
+ for (int i = 0; i < intermediateFileMergers.length; i++) {
+ intermediateFileMergers[i].finish();
+ }
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException(e);
+ }
+
+ Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
+ for (int i = 0; i < sortDataRows.length; i++) {
+ batchIterator[i] =
+ new MergedDataIterator(sortParameterArray[i], batchSize, intermediateFileMergers[i]);
+ }
+
+ return batchIterator;
+ }
+
+ private UnsafeSingleThreadFinalSortFilesMerger getFinalMerger(SortParameters sortParameters) {
+ String[] storeLocation = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()),
+ sortParameters.getSegmentId() + "", false, false);
+ // Set the data file location
+ String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation,
+ File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
+ return new UnsafeSingleThreadFinalSortFilesMerger(sortParameters, dataFolderLocation);
+ }
+
+ @Override public void close() {
+ for (int i = 0; i < intermediateFileMergers.length; i++) {
+ intermediateFileMergers[i].close();
+ }
+ }
+
+ /**
+ * Below method will be used to process data to next step
+ */
+ private boolean processRowToNextStep(UnsafeSortDataRows[] sortDataRows, SortParameters parameters)
+ throws CarbonDataLoadingException {
+ if (null == sortDataRows || sortDataRows.length == 0) {
+ LOGGER.info("Record Processed For table: " + parameters.getTableName());
+ LOGGER.info("Number of Records was Zero");
+ String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
+ LOGGER.info(logMessage);
+ return false;
+ }
+
+ try {
+ for (int i = 0; i < sortDataRows.length; i++) {
+ // start sorting
+ sortDataRows[i].startSorting();
+ }
+ // check any more rows are present
+ LOGGER.info("Record Processed For table: " + parameters.getTableName());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
+ return false;
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ }
+
+ private void setTempLocation(SortParameters parameters) {
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(parameters.getDatabaseName(), parameters.getTableName(),
+ parameters.getTaskNo(), parameters.getSegmentId(),
+ false, false);
+ String[] tmpLoc = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
+ CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
+ LOGGER.error("set temp location: " + StringUtils.join(tmpLoc, ", "));
+ parameters.setTempFileLocation(tmpLoc);
+ }
+
+ /**
+ * This thread iterates the iterator and adds the rows to @{@link UnsafeSortDataRows}
+ */
+ private static class SortIteratorThread implements Runnable {
+
+ private Iterator<CarbonRowBatch> iterator;
+
+ private UnsafeSortDataRows[] sortDataRows;
+ private AtomicLong rowCounter;
+ private List<AtomicLong> insideRowCounterList;
+ private ThreadStatusObserver threadStatusObserver;
+
+ public SortIteratorThread(Iterator<CarbonRowBatch> iterator,
+ UnsafeSortDataRows[] sortDataRows, AtomicLong rowCounter,
+ List<AtomicLong> insideRowCounterList,
+ ThreadStatusObserver threadStatusObserver) {
+ this.iterator = iterator;
+ this.sortDataRows = sortDataRows;
+ this.rowCounter = rowCounter;
+ this.insideRowCounterList = insideRowCounterList;
+ this.threadStatusObserver = threadStatusObserver;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (iterator.hasNext()) {
+ CarbonRowBatch batch = iterator.next();
+ int i = 0;
+ while (batch.hasNext()) {
+ CarbonRow row = batch.next();
+ if (row != null) {
+ UnsafeSortDataRows sortDataRow = sortDataRows[row.getRangeId()];
+ synchronized (sortDataRow) {
+ rowCounter.getAndIncrement();
+ insideRowCounterList.get(row.getRangeId()).getAndIncrement();
+ sortDataRow.addRow(row.getData());
+ }
+ }
+ }
+ }
+ LOGGER.info("Rows processed by each range: " + insideRowCounterList);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ this.threadStatusObserver.notifyFailed(e);
+ }
+ }
+ }
+
+ private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
+
+ private SortParameters sortParameters;
+
+ private int batchSize;
+
+ private boolean firstRow;
+
+ private UnsafeIntermediateMerger intermediateMerger;
+
+ public MergedDataIterator(SortParameters sortParameters, int batchSize,
+ UnsafeIntermediateMerger intermediateMerger) {
+ this.sortParameters = sortParameters;
+ this.batchSize = batchSize;
+ this.intermediateMerger = intermediateMerger;
+ this.firstRow = true;
+ }
+
+ private UnsafeSingleThreadFinalSortFilesMerger finalMerger;
+
+ @Override public boolean hasNext() {
+ if (firstRow) {
+ firstRow = false;
+ finalMerger = getFinalMerger(sortParameters);
+ List<UnsafeCarbonRowPage> rowPages = intermediateMerger.getRowPages();
+ finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]),
+ intermediateMerger.getMergedPages());
+ }
+ return finalMerger.hasNext();
+ }
+
+ @Override public CarbonRowBatch next() {
+ int counter = 0;
+ CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
+ while (finalMerger.hasNext() && counter < batchSize) {
+ rowBatch.addRow(new CarbonRow(finalMerger.next()));
+ counter++;
+ }
+ return rowBatch;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index 5d038d3..eaa858e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -366,9 +366,9 @@ public class UnsafeSortDataRows {
// create a new file and pick a temp directory randomly every time
String tmpDir = parameters.getTempFileLocation()[
new Random().nextInt(parameters.getTempFileLocation().length)];
- File sortTempFile = new File(
- tmpDir + File.separator + parameters.getTableName()
- + System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
+ File sortTempFile = new File(tmpDir + File.separator + parameters.getTableName()
+ + '_' + parameters.getRangeId() + '_' + System.nanoTime()
+ + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(page, sortTempFile);
LOGGER.info("Time taken to sort row page with size" + page.getBuffer().getActualSize()
+ " and write is: " + (System.currentTimeMillis() - startTime) + ": location:"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
index 0d24e01..104f3f5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -112,9 +112,9 @@ public class UnsafeIntermediateMerger {
String[] tempFileLocations = parameters.getTempFileLocation();
String targetLocation = tempFileLocations[new Random().nextInt(tempFileLocations.length)];
- File file = new File(
- targetLocation + File.separator + parameters.getTableName() + System
- .nanoTime() + CarbonCommonConstants.MERGERD_EXTENSION);
+ File file = new File(targetLocation + File.separator + parameters.getTableName()
+ + '_' + parameters.getRangeId() + '_' + System.nanoTime()
+ + CarbonCommonConstants.MERGERD_EXTENSION);
UnsafeIntermediateFileMerger merger =
new UnsafeIntermediateFileMerger(parameters, intermediateFiles, file);
mergerTask.add(executorService.submit(merger));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index 64f3c25..b1dc156 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -147,19 +147,20 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
}
private List<File> getFilesToMergeSort() {
+ // this can be partitionId, bucketId or rangeId, let's call it rangeId
+ final int rangeId = parameters.getRangeId();
+
FileFilter fileFilter = new FileFilter() {
public boolean accept(File pathname) {
- return pathname.getName().startsWith(tableName);
+ return pathname.getName().startsWith(tableName + '_' + rangeId);
}
};
// get all the merged files
List<File> files = new ArrayList<File>(tempFileLocation.length);
- for (String tempLoc : tempFileLocation)
- {
+ for (String tempLoc : tempFileLocation) {
File[] subFiles = new File(tempLoc).listFiles(fileFilter);
- if (null != subFiles && subFiles.length > 0)
- {
+ if (null != subFiles && subFiles.length > 0) {
files.addAll(Arrays.asList(subFiles));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
index 90a340d..72a8c25 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorStepImpl.java
@@ -19,21 +19,33 @@ package org.apache.carbondata.processing.loading.steps;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.SortColumnRangeInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.loading.BadRecordsLogger;
import org.apache.carbondata.processing.loading.BadRecordsLoggerProvider;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.DataField;
+import org.apache.carbondata.processing.loading.converter.BadRecordLogHolder;
+import org.apache.carbondata.processing.loading.converter.FieldConverter;
import org.apache.carbondata.processing.loading.converter.RowConverter;
import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl;
+import org.apache.carbondata.processing.loading.partition.Partitioner;
+import org.apache.carbondata.processing.loading.partition.impl.HashPartitionerImpl;
+import org.apache.carbondata.processing.loading.partition.impl.RangePartitionerImpl;
+import org.apache.carbondata.processing.loading.partition.impl.RawRowComparator;
import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
import org.apache.carbondata.processing.util.CarbonBadRecordUtil;
+import org.apache.commons.lang3.StringUtils;
+
/**
* Replace row data fields with dictionary values if column is configured dictionary encoded.
* And nondictionary columns as well as complex columns will be converted to byte[].
@@ -41,7 +53,10 @@ import org.apache.carbondata.processing.util.CarbonBadRecordUtil;
public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorStep {
private List<RowConverter> converters;
+ private Partitioner<CarbonRow> partitioner;
private BadRecordsLogger badRecordLogger;
+ private boolean isSortColumnRangeEnabled = false;
+ private boolean isBucketColumnEnabled = false;
public DataConverterProcessorStepImpl(CarbonDataLoadConfiguration configuration,
AbstractDataLoadProcessorStep child) {
@@ -64,6 +79,81 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
configuration.setCardinalityFinder(converter);
converters.add(converter);
converter.initialize();
+
+ if (null != configuration.getBucketingInfo()) {
+ this.isBucketColumnEnabled = true;
+ initializeBucketColumnPartitioner();
+ } else if (null != configuration.getSortColumnRangeInfo()) {
+ this.isSortColumnRangeEnabled = true;
+ initializeSortColumnRangesPartitioner();
+ }
+ }
+
+ /**
+ * initialize partitioner for bucket column
+ */
+ private void initializeBucketColumnPartitioner() {
+ List<Integer> indexes = new ArrayList<>();
+ List<ColumnSchema> columnSchemas = new ArrayList<>();
+ DataField[] inputDataFields = getOutput();
+ BucketingInfo bucketingInfo = configuration.getBucketingInfo();
+ for (int i = 0; i < inputDataFields.length; i++) {
+ for (int j = 0; j < bucketingInfo.getListOfColumns().size(); j++) {
+ if (inputDataFields[i].getColumn().getColName()
+ .equals(bucketingInfo.getListOfColumns().get(j).getColumnName())) {
+ indexes.add(i);
+ columnSchemas.add(inputDataFields[i].getColumn().getColumnSchema());
+ break;
+ }
+ }
+ }
+
+ // hash partitioner to dispatch rows by bucket column
+ this.partitioner =
+ new HashPartitionerImpl(indexes, columnSchemas, bucketingInfo.getNumOfRanges());
+ }
+
+
+ /**
+ * initialize partitioner for sort column ranges
+ */
+ private void initializeSortColumnRangesPartitioner() {
+ // convert user specified sort-column ranges
+ SortColumnRangeInfo sortColumnRangeInfo = configuration.getSortColumnRangeInfo();
+ int rangeValueCnt = sortColumnRangeInfo.getUserSpecifiedRanges().length;
+ CarbonRow[] convertedSortColumnRanges = new CarbonRow[rangeValueCnt];
+ for (int i = 0; i < rangeValueCnt; i++) {
+ Object[] fakeOriginRow = new Object[configuration.getDataFields().length];
+ String[] oneBound = StringUtils.splitPreserveAllTokens(
+ sortColumnRangeInfo.getUserSpecifiedRanges()[i], sortColumnRangeInfo.getSeparator(), -1);
+ // set the corresponding sort column
+ int j = 0;
+ for (int colIdx : sortColumnRangeInfo.getSortColumnIndex()) {
+ fakeOriginRow[colIdx] = oneBound[j++];
+ }
+ CarbonRow fakeCarbonRow = new CarbonRow(fakeOriginRow);
+ convertFakeRow(fakeCarbonRow, sortColumnRangeInfo);
+ convertedSortColumnRanges[i] = fakeCarbonRow;
+ }
+ // sort the range bounds (sort in carbon is a little different from what we think)
+ Arrays.sort(convertedSortColumnRanges,
+ new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(),
+ sortColumnRangeInfo.getIsSortColumnNoDict()));
+
+ // range partitioner to dispatch rows by sort columns
+ this.partitioner = new RangePartitionerImpl(convertedSortColumnRanges,
+ new RawRowComparator(sortColumnRangeInfo.getSortColumnIndex(),
+ sortColumnRangeInfo.getIsSortColumnNoDict()));
+ }
+
+ // only convert sort column fields
+ private void convertFakeRow(CarbonRow fakeRow, SortColumnRangeInfo sortColumnRangeInfo) {
+ FieldConverter[] fieldConverters = converters.get(0).getFieldConverters();
+ BadRecordLogHolder logHolder = new BadRecordLogHolder();
+ logHolder.setLogged(false);
+ for (int colIdx : sortColumnRangeInfo.getSortColumnIndex()) {
+ fieldConverters[colIdx].convert(fakeRow, logHolder);
+ }
}
/**
@@ -102,6 +192,10 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
protected CarbonRowBatch processRowBatch(CarbonRowBatch rowBatch, RowConverter localConverter) {
while (rowBatch.hasNext()) {
CarbonRow convertRow = localConverter.convert(rowBatch.next());
+ if (isSortColumnRangeEnabled || isBucketColumnEnabled) {
+ short rangeNumber = (short) partitioner.getPartition(convertRow);
+ convertRow.setRangeId(rangeNumber);
+ }
rowBatch.setPreviousRow(convertRow);
}
rowCounter.getAndAdd(rowBatch.getSize());
@@ -134,6 +228,12 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
@Override protected String getStepName() {
- return "Data Converter";
+ if (isBucketColumnEnabled) {
+ return "Data Converter with Bucketing";
+ } else if (isSortColumnRangeEnabled) {
+ return "Data Converter with sort column range";
+ } else {
+ return "Data Converter";
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorWithBucketingStepImpl.java
deleted file mode 100644
index a1181c9..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.steps;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.metadata.schema.BucketingInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
-import org.apache.carbondata.processing.loading.BadRecordsLogger;
-import org.apache.carbondata.processing.loading.BadRecordsLoggerProvider;
-import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.loading.DataField;
-import org.apache.carbondata.processing.loading.converter.RowConverter;
-import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl;
-import org.apache.carbondata.processing.loading.partition.Partitioner;
-import org.apache.carbondata.processing.loading.partition.impl.HashPartitionerImpl;
-import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
-import org.apache.carbondata.processing.util.CarbonBadRecordUtil;
-
-/**
- * Replace row data fields with dictionary values if column is configured dictionary encoded.
- * And nondictionary columns as well as complex columns will be converted to byte[].
- */
-public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoadProcessorStep {
-
- private List<RowConverter> converters;
-
- private Partitioner<Object[]> partitioner;
-
- private BadRecordsLogger badRecordLogger;
-
- public DataConverterProcessorWithBucketingStepImpl(CarbonDataLoadConfiguration configuration,
- AbstractDataLoadProcessorStep child) {
- super(configuration, child);
- }
-
- @Override
- public DataField[] getOutput() {
- return child.getOutput();
- }
-
- @Override
- public void initialize() throws IOException {
- super.initialize();
- child.initialize();
- converters = new ArrayList<>();
- badRecordLogger = BadRecordsLoggerProvider.createBadRecordLogger(configuration);
- RowConverter converter =
- new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);
- configuration.setCardinalityFinder(converter);
- converters.add(converter);
- converter.initialize();
- List<Integer> indexes = new ArrayList<>();
- List<ColumnSchema> columnSchemas = new ArrayList<>();
- DataField[] inputDataFields = getOutput();
- BucketingInfo bucketingInfo = configuration.getBucketingInfo();
- for (int i = 0; i < inputDataFields.length; i++) {
- for (int j = 0; j < bucketingInfo.getListOfColumns().size(); j++) {
- if (inputDataFields[i].getColumn().getColName()
- .equals(bucketingInfo.getListOfColumns().get(j).getColumnName())) {
- indexes.add(i);
- columnSchemas.add(inputDataFields[i].getColumn().getColumnSchema());
- break;
- }
- }
- }
- partitioner =
- new HashPartitionerImpl(indexes, columnSchemas, bucketingInfo.getNumberOfBuckets());
- }
-
- /**
- * Create the iterator using child iterator.
- *
- * @param childIter
- * @return new iterator with step specific processing.
- */
- @Override
- protected Iterator<CarbonRowBatch> getIterator(final Iterator<CarbonRowBatch> childIter) {
- return new CarbonIterator<CarbonRowBatch>() {
- RowConverter localConverter;
- private boolean first = true;
- @Override public boolean hasNext() {
- if (first) {
- first = false;
- localConverter = converters.get(0).createCopyForNewThread();
- converters.add(localConverter);
- }
- return childIter.hasNext();
- }
-
- @Override public CarbonRowBatch next() {
- return processRowBatch(childIter.next(), localConverter);
- }
- };
- }
-
- /**
- * Process the batch of rows as per the step logic.
- *
- * @param rowBatch
- * @return processed row.
- */
- protected CarbonRowBatch processRowBatch(CarbonRowBatch rowBatch, RowConverter localConverter) {
- while (rowBatch.hasNext()) {
- CarbonRow row = rowBatch.next();
- short bucketNumber = (short) partitioner.getPartition(row.getData());
- CarbonRow convertRow = localConverter.convert(row);
- convertRow.bucketNumber = bucketNumber;
- rowBatch.setPreviousRow(convertRow);
- }
- rowCounter.getAndAdd(rowBatch.getSize());
- // reuse the origin batch
- rowBatch.rewind();
- return rowBatch;
- }
-
- @Override
- protected CarbonRow processRow(CarbonRow row) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- if (!closed) {
- super.close();
- if (null != badRecordLogger) {
- badRecordLogger.closeStreams();
- CarbonBadRecordUtil.renameBadRecord(configuration);
- }
- if (converters != null) {
- for (RowConverter converter : converters) {
- converter.finish();
- }
- }
- }
- }
- @Override protected String getStepName() {
- return "Data Converter with Bucketing";
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
index 58009af..0467b11 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java
@@ -17,7 +17,15 @@
package org.apache.carbondata.processing.loading.steps;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -25,6 +33,7 @@ import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.processing.loading.AbstractDataLoadProcessorStep;
@@ -92,7 +101,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
.recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID,
System.currentTimeMillis());
+ ExecutorService rangeExecutorService = Executors.newFixedThreadPool(iterators.length,
+ new CarbonThreadFactory("WriterForwardPool: " + tableName));
+ List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length);
int i = 0;
+ // do this concurrently
for (Iterator<CarbonRowBatch> iterator : iterators) {
String[] storeLocation = getStoreLocation(tableIdentifier);
@@ -112,9 +125,19 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
if (!rowsNotExist) {
finish(dataHandler);
}
+ rangeExecutorServiceSubmitList.add(
+ rangeExecutorService.submit(new WriterForwarder(iterator, tableIdentifier, i)));
i++;
}
-
+ try {
+ rangeExecutorService.shutdown();
+ rangeExecutorService.awaitTermination(2, TimeUnit.DAYS);
+ for (int j = 0; j < rangeExecutorServiceSubmitList.size(); j++) {
+ rangeExecutorServiceSubmitList.get(j).get();
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ throw new CarbonDataWriterException(e);
+ }
} catch (CarbonDataWriterException e) {
LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
throw new CarbonDataLoadingException(
@@ -130,6 +153,51 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
return "Data Writer";
}
+ /**
+ * Used to forward rows to different ranges based on range id.
+ */
+ private final class WriterForwarder implements Callable<Void> {
+ private Iterator<CarbonRowBatch> insideRangeIterator;
+ private CarbonTableIdentifier tableIdentifier;
+ private int rangeId;
+
+ public WriterForwarder(Iterator<CarbonRowBatch> insideRangeIterator,
+ CarbonTableIdentifier tableIdentifier, int rangeId) {
+ this.insideRangeIterator = insideRangeIterator;
+ this.tableIdentifier = tableIdentifier;
+ this.rangeId = rangeId;
+ }
+
+ @Override public Void call() throws Exception {
+ LOGGER.info("Process writer forward for table " + tableIdentifier.getTableName()
+ + ", range: " + rangeId);
+ processRange(insideRangeIterator, tableIdentifier, rangeId);
+ return null;
+ }
+ }
+
+ private void processRange(Iterator<CarbonRowBatch> insideRangeIterator,
+ CarbonTableIdentifier tableIdentifier, int rangeId) {
+ String[] storeLocation = getStoreLocation(tableIdentifier);
+
+ CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, rangeId, 0);
+ CarbonFactHandler dataHandler = null;
+ boolean rowsNotExist = true;
+ while (insideRangeIterator.hasNext()) {
+ if (rowsNotExist) {
+ rowsNotExist = false;
+ dataHandler = CarbonFactHandlerFactory
+ .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ dataHandler.initialise();
+ }
+ processBatch(insideRangeIterator.next(), dataHandler);
+ }
+ if (!rowsNotExist) {
+ finish(dataHandler);
+ }
+ }
+
public void finish(CarbonFactHandler dataHandler) {
CarbonTableIdentifier tableIdentifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
index a4ac0ea..1a839a2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
@@ -119,9 +119,10 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
}
private List<File> getFilesToMergeSort() {
+ final int rangeId = sortParameters.getRangeId();
FileFilter fileFilter = new FileFilter() {
public boolean accept(File pathname) {
- return pathname.getName().startsWith(tableName);
+ return pathname.getName().startsWith(tableName + '_' + rangeId);
}
};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
index c7efbd9..a5caf7b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
@@ -212,9 +212,9 @@ public class SortDataRows {
// create new file and choose folder randomly
String[] tmpLocation = parameters.getTempFileLocation();
String locationChosen = tmpLocation[new Random().nextInt(tmpLocation.length)];
- File file = new File(
- locationChosen + File.separator + parameters.getTableName() +
- System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
+ File file = new File(locationChosen + File.separator + parameters.getTableName()
+ + '_' + parameters.getRangeId() + '_' + System.nanoTime()
+ + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(recordHolderList, this.entryCount, file);
}
@@ -325,8 +325,9 @@ public class SortDataRows {
String[] tmpFileLocation = parameters.getTempFileLocation();
String locationChosen = tmpFileLocation[new Random().nextInt(tmpFileLocation.length)];
File sortTempFile = new File(
- locationChosen + File.separator + parameters.getTableName() + System
- .nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
+ locationChosen + File.separator + parameters.getTableName()
+ + '_' + parameters.getRangeId() + '_' + System.nanoTime()
+ + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(recordHolderArray, recordHolderArray.length, sortTempFile);
// add sort temp filename to and arrayList. When the list size reaches 20 then
// intermediate merging of sort temp files will be triggered
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortIntermediateFileMerger.java
index 9c995a5..0e3f6bd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortIntermediateFileMerger.java
@@ -94,9 +94,9 @@ public class SortIntermediateFileMerger {
private void startIntermediateMerging(File[] intermediateFiles) {
int index = new Random().nextInt(parameters.getTempFileLocation().length);
String chosenTempDir = parameters.getTempFileLocation()[index];
- File file = new File(
- chosenTempDir + File.separator + parameters.getTableName() + System
- .nanoTime() + CarbonCommonConstants.MERGERD_EXTENSION);
+ File file = new File(chosenTempDir + File.separator + parameters.getTableName()
+ + '_' + parameters.getRangeId() + '_' + System.nanoTime()
+ + CarbonCommonConstants.MERGERD_EXTENSION);
IntermediateFileMerger merger = new IntermediateFileMerger(parameters, intermediateFiles, file);
mergerTask.add(executorService.submit(merger));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
index 4d31f87..4d333ed 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortParameters.java
@@ -119,6 +119,7 @@ public class SortParameters implements Serializable {
private int numberOfCores;
private int batchSortSizeinMb;
+ private int rangeId = 0;
public SortParameters getCopy() {
SortParameters parameters = new SortParameters();
@@ -147,6 +148,7 @@ public class SortParameters implements Serializable {
parameters.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
parameters.numberOfCores = numberOfCores;
parameters.batchSortSizeinMb = batchSortSizeinMb;
+ parameters.rangeId = rangeId;
return parameters;
}
@@ -429,6 +431,14 @@ public class SortParameters implements Serializable {
return parameters;
}
+ public int getRangeId() {
+ return rangeId;
+ }
+
+ public void setRangeId(int rangeId) {
+ this.rangeId = rangeId;
+ }
+
/**
* this method will set the boolean mapping for no dictionary sort columns
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index b795696..c0acadd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -167,12 +167,14 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
blockletProcessingCount = new AtomicInteger(0);
producerExecutorService = Executors.newFixedThreadPool(numberOfCores,
- new CarbonThreadFactory("ProducerPool:" + model.getTableName()));
+ new CarbonThreadFactory("ProducerPool:" + model.getTableName()
+ + ", range: " + model.getBucketId()));
producerExecutorServiceTaskList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
LOGGER.info("Initializing writer executors");
consumerExecutorService = Executors
- .newFixedThreadPool(1, new CarbonThreadFactory("ConsumerPool:" + model.getTableName()));
+ .newFixedThreadPool(1, new CarbonThreadFactory("ConsumerPool:" + model.getTableName()
+ + ", range: " + model.getBucketId()));
consumerExecutorServiceTaskList = new ArrayList<>(1);
semaphore = new Semaphore(numberOfCores);
tablePageList = new TablePageList();
[37/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
new file mode 100644
index 0000000..4c0e637
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.preaaggregate
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.execution.command._
+import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
+import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
+import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
+import org.apache.spark.sql.optimizer.CarbonFilters
+import org.apache.spark.sql.parser.CarbonSpark2SqlParser
+
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
+import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
+import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema}
+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
+
+/**
+ * Below helper class will be used to create pre-aggregate table
+ * and updating the parent table about the child table information
+ * It will be either success or nothing happen in case of failure:
+ * 1. failed to create pre aggregate table.
+ * 2. failed to update main table
+ *
+ */
+case class PreAggregateTableHelper(
+ var parentTable: CarbonTable,
+ dataMapName: String,
+ dataMapClassName: String,
+ dataMapProperties: java.util.Map[String, String],
+ queryString: String,
+ timeSeriesFunction: Option[String] = None,
+ ifNotExistsSet: Boolean = false) {
+
+ var loadCommand: CarbonLoadDataCommand = _
+
+ def initMeta(sparkSession: SparkSession): Seq[Row] = {
+ val dmProperties = dataMapProperties.asScala
+ val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(queryString)
+ val df = sparkSession.sql(updatedQuery)
+ val fieldRelationMap = PreAggregateUtil.validateActualSelectPlanAndGetAttributes(
+ df.logicalPlan, queryString)
+ val fields = fieldRelationMap.keySet.toSeq
+ val tableProperties = mutable.Map[String, String]()
+ dmProperties.foreach(t => tableProperties.put(t._1, t._2))
+
+ val selectTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
+ if (!parentTable.getTableName.equalsIgnoreCase(selectTable.getTableName)) {
+ throw new MalformedDataMapCommandException(
+ "Parent table name is different in select and create")
+ }
+ var neworder = Seq[String]()
+ val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala
+ parentOrder.foreach(parentcol =>
+ fields.filter(col => fieldRelationMap(col).aggregateFunction.isEmpty &&
+ parentcol.equals(fieldRelationMap(col).
+ columnTableRelationList.get(0).parentColumnName))
+ .map(cols => neworder :+= cols.column)
+ )
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, neworder.mkString(","))
+ tableProperties.put("sort_scope", parentTable.getTableInfo.getFactTable.
+ getTableProperties.asScala.getOrElse("sort_scope", CarbonCommonConstants
+ .LOAD_SORT_SCOPE_DEFAULT))
+ tableProperties
+ .put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
+ val tableIdentifier =
+ TableIdentifier(parentTable.getTableName + "_" + dataMapName,
+ Some(parentTable.getDatabaseName))
+ // prepare table model of the collected tokens
+ val tableModel: TableModel = new CarbonSpark2SqlParser().prepareTableModel(
+ ifNotExistPresent = ifNotExistsSet,
+ new CarbonSpark2SqlParser().convertDbNameToLowerCase(tableIdentifier.database),
+ tableIdentifier.table.toLowerCase,
+ fields,
+ Seq(),
+ tableProperties,
+ None,
+ isAlterFlow = false,
+ None)
+
+ // updating the relation identifier, this will be stored in child table
+ // which can be used during dropping of pre-aggreate table as parent table will
+ // also get updated
+ if(timeSeriesFunction != null) {
+ TimeSeriesUtil.validateTimeSeriesEventTime(dmProperties.toMap, parentTable)
+ TimeSeriesUtil.validateEventTimeColumnExitsInSelect(
+ fieldRelationMap,
+ dmProperties(TimeSeriesUtil.TIMESERIES_EVENTTIME))
+ TimeSeriesUtil.updateTimeColumnSelect(
+ fieldRelationMap,
+ dmProperties(TimeSeriesUtil.TIMESERIES_EVENTTIME),
+ timeSeriesFunction.get)
+ }
+ tableModel.parentTable = Some(parentTable)
+ tableModel.dataMapRelation = Some(fieldRelationMap)
+ val tablePath = if (dmProperties.contains("path")) {
+ dmProperties("path")
+ } else {
+ CarbonEnv.getTablePath(tableModel.databaseNameOp, tableModel.tableName)(sparkSession)
+ }
+ CarbonCreateTableCommand(TableNewProcessor(tableModel),
+ tableModel.ifNotExistsSet, Some(tablePath), isVisible = false).run(sparkSession)
+
+ val table = CarbonEnv.getCarbonTable(tableIdentifier)(sparkSession)
+ val tableInfo = table.getTableInfo
+
+ // child schema object will be saved on parent table schema
+ val childSchema = tableInfo.getFactTable.buildChildSchema(
+ dataMapName,
+ dataMapClassName,
+ tableInfo.getDatabaseName,
+ queryString,
+ "AGGREGATION")
+ dmProperties.foreach(f => childSchema.getProperties.put(f._1, f._2))
+
+ // updating the parent table about child table
+ PreAggregateUtil.updateMainTable(parentTable, childSchema, sparkSession)
+
+ // After updating the parent carbon table with data map entry extract the latest table object
+ // to be used in further create process.
+ parentTable = CarbonEnv.getCarbonTable(Some(parentTable.getDatabaseName),
+ parentTable.getTableName)(sparkSession)
+
+ val updatedLoadQuery = if (timeSeriesFunction != null) {
+ PreAggregateUtil.createTimeSeriesSelectQueryFromMain(
+ childSchema.getChildSchema,
+ parentTable.getTableName,
+ parentTable.getDatabaseName)
+ } else {
+ queryString
+ }
+ val dataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
+ updatedLoadQuery)).drop("preAggLoad")
+ loadCommand = PreAggregateUtil.createLoadCommandForChild(
+ childSchema.getChildSchema.getListOfColumns,
+ tableIdentifier,
+ dataFrame,
+ isOverwrite = false,
+ sparkSession = sparkSession)
+ loadCommand.processMetadata(sparkSession)
+ Seq.empty
+ }
+
+ def initData(sparkSession: SparkSession): Seq[Row] = {
+ // load child table if parent table has existing segments
+ // This will be used to check if the parent table has any segments or not. If not then no
+ // need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
+ // table.
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(parentTable, false)
+ val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
+ if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
+ load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
+ throw new UnsupportedOperationException(
+ "Cannot create pre-aggregate table when insert is in progress on main table")
+ } else if (loadAvailable.nonEmpty) {
+ // Passing segmentToLoad as * because we want to load all the segments into the
+ // pre-aggregate table even if the user has set some segments on the parent table.
+ loadCommand.dataFrame = Some(PreAggregateUtil
+ .getDataFrame(sparkSession, loadCommand.logicalPlan.get))
+ PreAggregateUtil.startDataLoadForDataMap(
+ TableIdentifier(parentTable.getTableName, Some(parentTable.getDatabaseName)),
+ segmentToLoad = "*",
+ validateSegments = true,
+ loadCommand,
+ isOverwrite = false,
+ sparkSession)
+ }
+ Seq.empty
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index dd42b50..b52d0e7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.execution.command.{ColumnTableRelation, DataMapField, Field}
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonRelation}
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.spark.sql.types.DataType
@@ -407,11 +407,11 @@ object PreAggregateUtil {
* Below method will be used to update the main table about the pre aggregate table information
* in case of any exception it will throw error so pre aggregate table creation will fail
*
- * @param childSchema
- * @param sparkSession
+ * @return the existing TableInfo object before updating, it can be used to recover if any
+ * operation failed later
*/
def updateMainTable(carbonTable: CarbonTable,
- childSchema: DataMapSchema, sparkSession: SparkSession): Unit = {
+ childSchema: DataMapSchema, sparkSession: SparkSession): TableInfo = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
LockUsage.DROP_TABLE_LOCK)
@@ -424,7 +424,7 @@ object PreAggregateUtil {
locks = acquireLock(dbName, tableName, locksToBeAcquired, carbonTable)
// get the latest carbon table and check for column existence
// read the latest schema file
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
@@ -436,12 +436,11 @@ object PreAggregateUtil {
throw new MetadataProcessException("DataMap name already exist")
}
wrapperTableInfo.getDataMapSchemaList.add(childSchema)
- val thriftTable = schemaConverter
- .fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName)
- updateSchemaInfo(carbonTable,
- thriftTable)(sparkSession)
- LOGGER.info(s"Parent table updated is successful for table" +
- s" $dbName.${childSchema.getRelationIdentifier.toString}")
+ val thriftTable = schemaConverter.fromWrapperToExternalTableInfo(
+ wrapperTableInfo, dbName, tableName)
+ updateSchemaInfo(carbonTable, thriftTable)(sparkSession)
+ LOGGER.info(s"Parent table updated is successful for table $dbName.$tableName")
+ thriftTableInfo
} catch {
case e: Exception =>
LOGGER.error(e, "Pre Aggregate Parent table update failed reverting changes")
@@ -450,7 +449,6 @@ object PreAggregateUtil {
// release lock after command execution completion
releaseLocks(locks)
}
- Seq.empty
}
/**
@@ -526,7 +524,7 @@ object PreAggregateUtil {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
carbonTable.getTableLastUpdatedTime
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
if (thriftTable.dataMapSchemas.size > numberOfChildSchema) {
metastore.revertTableSchemaForPreAggCreationFailure(
carbonTable.getAbsoluteTableIdentifier, thriftTable)(sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
index 064ba18..4962c3a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableAddColumnCommand.scala
@@ -63,7 +63,7 @@ private[sql] case class CarbonAlterTableAddColumnCommand(
OperationListenerBus.getInstance().fireEvent(alterTableAddColumnListener, operationContext)
// get the latest carbon table and check for column existence
// read the latest schema file
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter
.fromExternalToWrapperTableInfo(thriftTableInfo,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index 4ef0df9..c8f7ac7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -74,7 +74,7 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
}
// read the latest schema file
val tableInfo: org.apache.carbondata.format.TableInfo =
- metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ metastore.getThriftTableInfo(carbonTable)
// maintain the added column for schema evolution history
var addColumnSchema: org.apache.carbondata.format.ColumnSchema = null
var deletedColumnSchema: org.apache.carbondata.format.ColumnSchema = null
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index 7bbefd7..a64bdb9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -100,7 +100,7 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
// read the latest schema file
val tableInfo: org.apache.carbondata.format.TableInfo =
- metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ metastore.getThriftTableInfo(carbonTable)
// maintain the deleted columns for schema evolution history
var deletedColumnSchema = ListBuffer[org.apache.carbondata.format.ColumnSchema]()
val columnSchemaList = tableInfo.fact_table.table_columns.asScala
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 2503fc3..05c0059 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -108,7 +108,7 @@ private[sql] case class CarbonAlterTableRenameCommand(
sparkSession)
OperationListenerBus.getInstance().fireEvent(alterTableRenamePreEvent, operationContext)
val tableInfo: org.apache.carbondata.format.TableInfo =
- metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ metastore.getThriftTableInfo(carbonTable)
val schemaEvolutionEntry = new SchemaEvolutionEntry(System.currentTimeMillis)
schemaEvolutionEntry.setTableName(newTableName)
timeStamp = System.currentTimeMillis()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index 2b35416..6fd0820 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -63,13 +63,13 @@ object TimeSeriesUtil {
* @return whether find only one granularity
*/
def validateTimeSeriesGranularity(
- dmProperties: Map[String, String],
+ dmProperties: java.util.Map[String, String],
dmClassName: String): Boolean = {
var isFound = false
// 1. granularity only support one
for (granularity <- Granularity.values()) {
- if (dmProperties.get(granularity.getName).isDefined) {
+ if (dmProperties.containsKey(granularity.getName)) {
if (isFound) {
throw new MalformedDataMapCommandException(
s"Only one granularity level can be defined")
@@ -104,14 +104,14 @@ object TimeSeriesUtil {
* @return key and value tuple
*/
def getTimeSeriesGranularityDetails(
- dmProperties: Map[String, String],
+ dmProperties: java.util.Map[String, String],
dmClassName: String): (String, String) = {
val defaultValue = "1"
for (granularity <- Granularity.values()) {
- if (dmProperties.get(granularity.getName).isDefined &&
- dmProperties.get(granularity.getName).get.trim.equalsIgnoreCase(defaultValue)) {
- return (granularity.toString.toLowerCase, dmProperties.get(granularity.getName).get)
+ if (dmProperties.containsKey(granularity.getName) &&
+ dmProperties.get(granularity.getName).trim.equalsIgnoreCase(defaultValue)) {
+ return (granularity.toString.toLowerCase, dmProperties.get(granularity.getName))
}
}
@@ -194,8 +194,8 @@ object TimeSeriesUtil {
* @param timeSeriesColumn
* timeseries column name
*/
- def updateTimeColumnSelect(fieldMapping: scala.collection.mutable
- .LinkedHashMap[Field, DataMapField],
+ def updateTimeColumnSelect(
+ fieldMapping: scala.collection.mutable.LinkedHashMap[Field, DataMapField],
timeSeriesColumn: String,
timeSeriesFunction: String) : Any = {
val isTimeSeriesColumnExits = fieldMapping
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index fd09e48..b3438a4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -523,8 +523,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] =
metadata.carbonTables
- override def getThriftTableInfo(carbonTable: CarbonTable)
- (sparkSession: SparkSession): TableInfo = {
+ override def getThriftTableInfo(carbonTable: CarbonTable): TableInfo = {
val tableMetadataFile = CarbonTablePath.getSchemaFilePath(carbonTable.getTablePath)
CarbonUtil.readSchemaFile(tableMetadataFile)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index 5e242b7..96ef473 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -96,8 +96,7 @@ class CarbonHiveMetaStore extends CarbonFileMetastore {
Seq()
}
- override def getThriftTableInfo(carbonTable: CarbonTable)
- (sparkSession: SparkSession): format.TableInfo = {
+ override def getThriftTableInfo(carbonTable: CarbonTable): format.TableInfo = {
val schemaConverter = new ThriftWrapperSchemaConverterImpl
schemaConverter.fromWrapperToExternalTableInfo(carbonTable.getTableInfo,
carbonTable.getDatabaseName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index fc5bdac..3f1ebbc 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -143,8 +143,9 @@ trait CarbonMetaStore {
def listAllTables(sparkSession: SparkSession): Seq[CarbonTable]
- def getThriftTableInfo(carbonTable: CarbonTable)(sparkSession: SparkSession):
- org.apache.carbondata.format.TableInfo
+ def getThriftTableInfo(
+ carbonTable: CarbonTable
+ ): org.apache.carbondata.format.TableInfo
def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index aaa87a3..9c2c7e7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -192,7 +192,7 @@ object AlterTableUtil {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val fileType = FileFactory.getFileType(tablePath)
if (FileFactory.isFileExist(tablePath, fileType)) {
- val tableInfo = metastore.getThriftTableInfo(oldCarbonTable)(sparkSession)
+ val tableInfo = metastore.getThriftTableInfo(oldCarbonTable)
val evolutionEntryList = tableInfo.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -221,7 +221,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -246,7 +246,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -277,7 +277,7 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
- val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTable)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -326,7 +326,7 @@ object AlterTableUtil {
carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
// get the latest carbon table
// read the latest schema file
- val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(
thriftTableInfo,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index f514074..be8d4d7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -31,13 +31,13 @@ import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.processing.store.TablePage;
/**
- * It is for writing DataMap for one table
+ * It is for writing IndexDataMap for one table
*/
public class DataMapWriterListener {
@@ -55,7 +55,7 @@ public class DataMapWriterListener {
List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable);
if (tableDataMaps != null) {
for (TableDataMap tableDataMap : tableDataMaps) {
- DataMapFactory factory = tableDataMap.getDataMapFactory();
+ IndexDataMapFactory factory = tableDataMap.getIndexDataMapFactory();
register(factory, segmentId, dataWritePath);
}
}
@@ -64,7 +64,7 @@ public class DataMapWriterListener {
/**
* Register a AbstractDataMapWriter
*/
- private void register(DataMapFactory factory, String segmentId, String dataWritePath) {
+ private void register(IndexDataMapFactory factory, String segmentId, String dataWritePath) {
assert (factory != null);
assert (segmentId != null);
DataMapMeta meta = factory.getMeta();
[26/54] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove
carbon-spark dependency in store-sdk module
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
new file mode 100644
index 0000000..fbb93b6
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.model;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.Maps;
+import org.apache.carbondata.common.Strings;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.constants.LoggerAction;
+import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
+import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
+import org.apache.carbondata.processing.util.TableOptionConstant;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Builder for {@link CarbonLoadModel}
+ */
+@InterfaceAudience.Developer
+public class CarbonLoadModelBuilder {
+
+ private CarbonTable table;
+
+ public CarbonLoadModelBuilder(CarbonTable table) {
+ this.table = table;
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param options Load options from user input
+ * @return a new CarbonLoadModel instance
+ */
+ public CarbonLoadModel build(
+ Map<String, String> options) throws InvalidLoadOptionException, IOException {
+ Map<String, String> optionsFinal = LoadOption.fillOptionWithDefaultValue(options);
+ optionsFinal.put("sort_scope", "no_sort");
+ if (!options.containsKey("fileheader")) {
+ List<CarbonColumn> csvHeader = table.getCreateOrderColumn(table.getTableName());
+ String[] columns = new String[csvHeader.size()];
+ for (int i = 0; i < columns.length; i++) {
+ columns[i] = csvHeader.get(i).getColName();
+ }
+ optionsFinal.put("fileheader", Strings.mkString(columns, ","));
+ }
+ CarbonLoadModel model = new CarbonLoadModel();
+
+ // we have provided 'fileheader', so it hadoopConf can be null
+ build(options, optionsFinal, model, null);
+
+ // set default values
+ model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ model.setUseOnePass(Boolean.parseBoolean(Maps.getOrDefault(options, "onepass", "false")));
+ model.setDictionaryServerHost(Maps.getOrDefault(options, "dicthost", null));
+ try {
+ model.setDictionaryServerPort(Integer.parseInt(Maps.getOrDefault(options, "dictport", "-1")));
+ } catch (NumberFormatException e) {
+ throw new InvalidLoadOptionException(e.getMessage());
+ }
+ return model;
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param options Load options from user input
+ * @param optionsFinal Load options that populated with default values for optional options
+ * @param carbonLoadModel The output load model
+ * @param hadoopConf hadoopConf is needed to read CSV header if there 'fileheader' is not set in
+ * user provided load options
+ */
+ public void build(
+ Map<String, String> options,
+ Map<String, String> optionsFinal,
+ CarbonLoadModel carbonLoadModel,
+ Configuration hadoopConf) throws InvalidLoadOptionException, IOException {
+ carbonLoadModel.setTableName(table.getTableName());
+ carbonLoadModel.setDatabaseName(table.getDatabaseName());
+ carbonLoadModel.setTablePath(table.getTablePath());
+ carbonLoadModel.setTableName(table.getTableName());
+ CarbonDataLoadSchema dataLoadSchema = new CarbonDataLoadSchema(table);
+ // Need to fill dimension relation
+ carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema);
+ String sort_scope = optionsFinal.get("sort_scope");
+ String single_pass = optionsFinal.get("single_pass");
+ String bad_records_logger_enable = optionsFinal.get("bad_records_logger_enable");
+ String bad_records_action = optionsFinal.get("bad_records_action");
+ String bad_record_path = optionsFinal.get("bad_record_path");
+ String global_sort_partitions = optionsFinal.get("global_sort_partitions");
+ String timestampformat = optionsFinal.get("timestampformat");
+ String dateFormat = optionsFinal.get("dateformat");
+ String delimeter = optionsFinal.get("delimiter");
+ String complex_delimeter_level1 = optionsFinal.get("complex_delimiter_level_1");
+ String complex_delimeter_level2 = optionsFinal.get("complex_delimiter_level_2");
+ String all_dictionary_path = optionsFinal.get("all_dictionary_path");
+ String column_dict = optionsFinal.get("columndict");
+ validateDateTimeFormat(timestampformat, "TimestampFormat");
+ validateDateTimeFormat(dateFormat, "DateFormat");
+ validateSortScope(sort_scope);
+
+ if (Boolean.parseBoolean(bad_records_logger_enable) ||
+ LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
+ bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path);
+ if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
+ throw new InvalidLoadOptionException("Invalid bad records location.");
+ }
+ }
+ carbonLoadModel.setBadRecordsLocation(bad_record_path);
+
+ validateGlobalSortPartitions(global_sort_partitions);
+ carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal.get("escapechar"), "\\"));
+ carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal.get("quotechar"), "\""));
+ carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal.get("commentchar"), "#"));
+
+ // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
+ // we should use table schema to generate file header.
+ String fileHeader = optionsFinal.get("fileheader");
+ String headerOption = options.get("header");
+ if (headerOption != null) {
+ if (!headerOption.equalsIgnoreCase("true") &&
+ !headerOption.equalsIgnoreCase("false")) {
+ throw new InvalidLoadOptionException(
+ "'header' option should be either 'true' or 'false'.");
+ }
+ // whether the csv file has file header, the default value is true
+ if (Boolean.valueOf(headerOption)) {
+ if (!StringUtils.isEmpty(fileHeader)) {
+ throw new InvalidLoadOptionException(
+ "When 'header' option is true, 'fileheader' option is not required.");
+ }
+ } else {
+ if (StringUtils.isEmpty(fileHeader)) {
+ List<CarbonColumn> columns = table.getCreateOrderColumn(table.getTableName());
+ String[] columnNames = new String[columns.size()];
+ for (int i = 0; i < columnNames.length; i++) {
+ columnNames[i] = columns.get(i).getColName();
+ }
+ fileHeader = Strings.mkString(columnNames, ",");
+ }
+ }
+ }
+
+ carbonLoadModel.setTimestampformat(timestampformat);
+ carbonLoadModel.setDateFormat(dateFormat);
+ carbonLoadModel.setDefaultTimestampFormat(
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+
+ carbonLoadModel.setDefaultDateFormat(
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_DATE_FORMAT,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
+
+ carbonLoadModel.setSerializationNullFormat(
+ TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName() + "," +
+ optionsFinal.get("serialization_null_format"));
+
+ carbonLoadModel.setBadRecordsLoggerEnable(
+ TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName() + "," + bad_records_logger_enable);
+
+ carbonLoadModel.setBadRecordsAction(
+ TableOptionConstant.BAD_RECORDS_ACTION.getName() + "," + bad_records_action.toUpperCase());
+
+ carbonLoadModel.setIsEmptyDataBadRecord(
+ DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
+ optionsFinal.get("is_empty_data_bad_record"));
+
+ carbonLoadModel.setSkipEmptyLine(optionsFinal.get("skip_empty_line"));
+
+ carbonLoadModel.setSortScope(sort_scope);
+ carbonLoadModel.setBatchSortSizeInMb(optionsFinal.get("batch_sort_size_inmb"));
+ carbonLoadModel.setGlobalSortPartitions(global_sort_partitions);
+ carbonLoadModel.setUseOnePass(Boolean.parseBoolean(single_pass));
+
+ if (delimeter.equalsIgnoreCase(complex_delimeter_level1) ||
+ complex_delimeter_level1.equalsIgnoreCase(complex_delimeter_level2) ||
+ delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
+ throw new InvalidLoadOptionException("Field Delimiter and Complex types delimiter are same");
+ } else {
+ carbonLoadModel.setComplexDelimiterLevel1(
+ CarbonUtil.delimiterConverter(complex_delimeter_level1));
+ carbonLoadModel.setComplexDelimiterLevel2(
+ CarbonUtil.delimiterConverter(complex_delimeter_level2));
+ }
+ // set local dictionary path, and dictionary file extension
+ carbonLoadModel.setAllDictPath(all_dictionary_path);
+ carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter));
+ carbonLoadModel.setCsvHeader(fileHeader);
+ carbonLoadModel.setColDictFilePath(column_dict);
+ carbonLoadModel.setCsvHeaderColumns(
+ LoadOption.getCsvHeaderColumns(carbonLoadModel, hadoopConf));
+
+ int validatedMaxColumns = validateMaxColumns(
+ carbonLoadModel.getCsvHeaderColumns(),
+ optionsFinal.get("maxcolumns"));
+
+ carbonLoadModel.setMaxColumns(String.valueOf(validatedMaxColumns));
+ carbonLoadModel.readAndSetLoadMetadataDetails();
+ }
+
+ private int validateMaxColumns(String[] csvHeaders, String maxColumns)
+ throws InvalidLoadOptionException {
+ /*
+ User configures both csvheadercolumns, maxcolumns,
+ if csvheadercolumns >= maxcolumns, give error
+ if maxcolumns > threashold, give error
+ User configures csvheadercolumns
+ if csvheadercolumns >= maxcolumns(default) then maxcolumns = csvheadercolumns+1
+ if csvheadercolumns >= threashold, give error
+ User configures nothing
+ if csvheadercolumns >= maxcolumns(default) then maxcolumns = csvheadercolumns+1
+ if csvheadercolumns >= threashold, give error
+ */
+ int columnCountInSchema = csvHeaders.length;
+ int maxNumberOfColumnsForParsing = 0;
+ Integer maxColumnsInt = getMaxColumnValue(maxColumns);
+ if (maxColumnsInt != null) {
+ if (columnCountInSchema >= maxColumnsInt) {
+ throw new InvalidLoadOptionException(
+ "csv headers should be less than the max columns " + maxColumnsInt);
+ } else if (maxColumnsInt > CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
+ throw new InvalidLoadOptionException(
+ "max columns cannot be greater than the threshold value: " +
+ CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING);
+ } else {
+ maxNumberOfColumnsForParsing = maxColumnsInt;
+ }
+ } else if (columnCountInSchema >= CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
+ throw new InvalidLoadOptionException(
+ "csv header columns should be less than max threashold: " +
+ CSVInputFormat.THRESHOLD_MAX_NUMBER_OF_COLUMNS_FOR_PARSING);
+ } else if (columnCountInSchema >= CSVInputFormat.DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING) {
+ maxNumberOfColumnsForParsing = columnCountInSchema + 1;
+ } else {
+ maxNumberOfColumnsForParsing = CSVInputFormat.DEFAULT_MAX_NUMBER_OF_COLUMNS_FOR_PARSING;
+ }
+ return maxNumberOfColumnsForParsing;
+ }
+
+ private Integer getMaxColumnValue(String maxColumn) {
+ return (maxColumn == null) ? null : Integer.parseInt(maxColumn);
+ }
+
+ /**
+ * validates both timestamp and date for illegal values
+ */
+ private void validateDateTimeFormat(String dateTimeLoadFormat, String dateTimeLoadOption)
+ throws InvalidLoadOptionException {
+ // allowing empty value to be configured for dateformat option.
+ if (dateTimeLoadFormat != null && !dateTimeLoadFormat.trim().equalsIgnoreCase("")) {
+ try {
+ new SimpleDateFormat(dateTimeLoadFormat);
+ } catch (IllegalArgumentException e) {
+ throw new InvalidLoadOptionException(
+ "Error: Wrong option: " + dateTimeLoadFormat + " is provided for option "
+ + dateTimeLoadOption);
+ }
+ }
+ }
+
+ private void validateSortScope(String sortScope) throws InvalidLoadOptionException {
+ if (sortScope != null) {
+ // Don't support use global sort on partitioned table.
+ if (table.getPartitionInfo(table.getTableName()) != null &&
+ sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString())) {
+ throw new InvalidLoadOptionException("Don't support use global sort on partitioned table.");
+ }
+ }
+ }
+
+ private void validateGlobalSortPartitions(String globalSortPartitions)
+ throws InvalidLoadOptionException {
+ if (globalSortPartitions != null) {
+ try {
+ int num = Integer.parseInt(globalSortPartitions);
+ if (num <= 0) {
+ throw new InvalidLoadOptionException("'GLOBAL_SORT_PARTITIONS' should be greater than 0");
+ }
+ } catch (NumberFormatException e) {
+ throw new InvalidLoadOptionException(e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * check whether using default value or not
+ */
+ private String checkDefaultValue(String value, String defaultValue) {
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ } else {
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
new file mode 100644
index 0000000..8ec93a9
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.model;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.common.Maps;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.carbondata.processing.util.CarbonLoaderUtil;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Provide utilities to populate loading options
+ */
+@InterfaceAudience.Developer
+public class LoadOption {
+
+ private static LogService LOG = LogServiceFactory.getLogService(LoadOption.class.getName());
+
+ /**
+ * Based on the input options, fill and return data loading options with default value
+ */
+ public static Map<String, String> fillOptionWithDefaultValue(
+ Map<String, String> options) throws InvalidLoadOptionException {
+ Map<String, String> optionsFinal = new HashMap<>();
+ optionsFinal.put("delimiter", Maps.getOrDefault(options, "delimiter", ","));
+ optionsFinal.put("quotechar", Maps.getOrDefault(options, "quotechar", "\""));
+ optionsFinal.put("fileheader", Maps.getOrDefault(options, "fileheader", ""));
+ optionsFinal.put("commentchar", Maps.getOrDefault(options, "commentchar", "#"));
+ optionsFinal.put("columndict", Maps.getOrDefault(options, "columndict", null));
+
+ optionsFinal.put(
+ "escapechar",
+ CarbonLoaderUtil.getEscapeChar(Maps.getOrDefault(options,"escapechar", "\\")));
+
+ optionsFinal.put(
+ "serialization_null_format",
+ Maps.getOrDefault(options, "serialization_null_format", "\\N"));
+
+ optionsFinal.put(
+ "bad_records_logger_enable",
+ Maps.getOrDefault(
+ options,
+ "bad_records_logger_enable",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)));
+
+ String badRecordActionValue = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
+ CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT);
+
+ optionsFinal.put(
+ "bad_records_action",
+ Maps.getOrDefault(
+ options,
+ "bad_records_action",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
+ badRecordActionValue)));
+
+ optionsFinal.put(
+ "is_empty_data_bad_record",
+ Maps.getOrDefault(
+ options,
+ "is_empty_data_bad_record",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)));
+
+ optionsFinal.put(
+ "skip_empty_line",
+ Maps.getOrDefault(
+ options,
+ "skip_empty_line",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
+
+ optionsFinal.put(
+ "all_dictionary_path",
+ Maps.getOrDefault(options, "all_dictionary_path", ""));
+
+ optionsFinal.put(
+ "complex_delimiter_level_1",
+ Maps.getOrDefault(options,"complex_delimiter_level_1", "\\$"));
+
+ optionsFinal.put(
+ "complex_delimiter_level_2",
+ Maps.getOrDefault(options, "complex_delimiter_level_2", "\\:"));
+
+ optionsFinal.put(
+ "dateformat",
+ Maps.getOrDefault(
+ options,
+ "dateformat",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)));
+
+ optionsFinal.put(
+ "timestampformat",
+ Maps.getOrDefault(
+ options,
+ "timestampformat",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)));
+
+ optionsFinal.put(
+ "global_sort_partitions",
+ Maps.getOrDefault(
+ options,
+ "global_sort_partitions",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
+ null)));
+
+ optionsFinal.put("maxcolumns", Maps.getOrDefault(options, "maxcolumns", null));
+
+ optionsFinal.put(
+ "batch_sort_size_inmb",
+ Maps.getOrDefault(
+ options,
+ "batch_sort_size_inmb",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
+ CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))));
+
+ optionsFinal.put(
+ "bad_record_path",
+ Maps.getOrDefault(
+ options,
+ "bad_record_path",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC,
+ CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))));
+
+ String useOnePass = Maps.getOrDefault(
+ options,
+ "single_pass",
+ CarbonProperties.getInstance().getProperty(
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
+ CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim().toLowerCase();
+
+ boolean singlePass;
+
+ if (useOnePass.equalsIgnoreCase("true")) {
+ singlePass = true;
+ } else {
+ // when single_pass = false and if either alldictionarypath
+ // or columnDict is configured the do not allow load
+ if (StringUtils.isNotEmpty(optionsFinal.get("all_dictionary_path")) ||
+ StringUtils.isNotEmpty(optionsFinal.get("columndict"))) {
+ throw new InvalidLoadOptionException(
+ "Can not use all_dictionary_path or columndict without single_pass.");
+ } else {
+ singlePass = false;
+ }
+ }
+
+ optionsFinal.put("single_pass", String.valueOf(singlePass));
+ return optionsFinal;
+ }
+
+ /**
+ * Return CSV header field names
+ */
+ public static String[] getCsvHeaderColumns(
+ CarbonLoadModel carbonLoadModel,
+ Configuration hadoopConf) throws IOException {
+ String delimiter;
+ if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter())) {
+ delimiter = CarbonCommonConstants.COMMA;
+ } else {
+ delimiter = CarbonUtil.delimiterConverter(carbonLoadModel.getCsvDelimiter());
+ }
+ String csvFile = null;
+ String csvHeader = carbonLoadModel.getCsvHeader();
+ String[] csvColumns;
+ if (StringUtils.isBlank(csvHeader)) {
+ // read header from csv file
+ csvFile = carbonLoadModel.getFactFilePath().split(",")[0];
+ csvHeader = CarbonUtil.readHeader(csvFile, hadoopConf);
+ if (StringUtils.isBlank(csvHeader)) {
+ throw new CarbonDataLoadingException("First line of the csv is not valid.");
+ }
+ String[] headers = csvHeader.toLowerCase().split(delimiter);
+ csvColumns = new String[headers.length];
+ for (int i = 0; i < csvColumns.length; i++) {
+ csvColumns[i] = headers[i].replaceAll("\"", "").trim();
+ }
+ } else {
+ String[] headers = csvHeader.toLowerCase().split(CarbonCommonConstants.COMMA);
+ csvColumns = new String[headers.length];
+ for (int i = 0; i < csvColumns.length; i++) {
+ csvColumns[i] = headers[i].trim();
+ }
+ }
+
+ if (!CarbonDataProcessorUtil.isHeaderValid(carbonLoadModel.getTableName(), csvColumns,
+ carbonLoadModel.getCarbonDataLoadSchema())) {
+ if (csvFile == null) {
+ LOG.error("CSV header in DDL is not proper."
+ + " Column names in schema and CSV header are not the same.");
+ throw new CarbonDataLoadingException(
+ "CSV header in DDL is not proper. Column names in schema and CSV header are "
+ + "not the same.");
+ } else {
+ LOG.error(
+ "CSV header in input file is not proper. Column names in schema and csv header are not "
+ + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));
+ throw new CarbonDataLoadingException(
+ "CSV header in input file is not proper. Column names in schema and csv header are not "
+ + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));
+ }
+ }
+ return csvColumns;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 70c26ab..5df39bc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -16,14 +16,9 @@
*/
package org.apache.carbondata.processing.util;
-import java.io.BufferedWriter;
-import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
import java.util.*;
import org.apache.carbondata.common.logging.LogService;
@@ -40,9 +35,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.fileoperations.FileWriteOperation;
import org.apache.carbondata.core.locks.CarbonLockUtil;
import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -60,9 +52,8 @@ import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.merger.NodeBlockRelation;
import org.apache.carbondata.processing.merger.NodeMultiBlockRelation;
-import static org.apache.carbondata.core.enums.EscapeSequences.*;
-import com.google.gson.Gson;
+import static org.apache.carbondata.core.enums.EscapeSequences.*;
public final class CarbonLoaderUtil {
@@ -561,17 +552,6 @@ public final class CarbonLoaderUtil {
}
/**
- * This method will divide the blocks among the nodes as per the data locality
- *
- * @param blockInfos
- * @return
- */
- public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos) {
- // -1 if number of nodes has to be decided based on block location information
- return nodeBlockMapping(blockInfos, -1);
- }
-
- /**
* the method returns the number of required executors
*
* @param blockInfos
@@ -946,25 +926,6 @@ public final class CarbonLoaderUtil {
CarbonUtil.checkAndCreateFolder(segmentFolder);
}
- /**
- * This will update the old table status details before clean files to the latest table status.
- * @param oldList
- * @param newList
- * @return
- */
- public static List<LoadMetadataDetails> updateLoadMetadataFromOldToNew(
- LoadMetadataDetails[] oldList, LoadMetadataDetails[] newList) {
-
- List<LoadMetadataDetails> newListMetadata =
- new ArrayList<LoadMetadataDetails>(Arrays.asList(newList));
- for (LoadMetadataDetails oldSegment : oldList) {
- if ("false".equalsIgnoreCase(oldSegment.getVisibility())) {
- newListMetadata.get(newListMetadata.indexOf(oldSegment)).setVisibility("false");
- }
- }
- return newListMetadata;
- }
-
/*
* This method will add data size and index size into tablestatus for each segment. And also
* returns the size of the segment.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java b/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
deleted file mode 100644
index c00cc86..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/util/DeleteLoadFolders.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.util;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.locks.CarbonLockFactory;
-import org.apache.carbondata.core.locks.ICarbonLock;
-import org.apache.carbondata.core.locks.LockUsage;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.SegmentFileStore;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
-import org.apache.carbondata.core.statusmanager.SegmentStatus;
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-public final class DeleteLoadFolders {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(DeleteLoadFolders.class.getName());
-
- private DeleteLoadFolders() {
-
- }
-
- /**
- * returns segment path
- *
- * @param identifier
- * @param oneLoad
- * @return
- */
- private static String getSegmentPath(AbsoluteTableIdentifier identifier,
- LoadMetadataDetails oneLoad) {
- String segmentId = oneLoad.getLoadName();
- return CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
- }
-
- public static void physicalFactAndMeasureMetadataDeletion(
- AbsoluteTableIdentifier absoluteTableIdentifier, String metadataPath, boolean isForceDelete,
- List<PartitionSpec> specs) {
- LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
- for (LoadMetadataDetails oneLoad : currentDetails) {
- if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
- try {
- if (oneLoad.getSegmentFile() != null) {
- SegmentFileStore
- .deleteSegment(absoluteTableIdentifier.getTablePath(), oneLoad.getSegmentFile(),
- specs);
- } else {
- String path = getSegmentPath(absoluteTableIdentifier, oneLoad);
- boolean status = false;
- if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
- CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
- CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile file) {
- return (CarbonTablePath.isCarbonDataFile(file.getName()) ||
- CarbonTablePath.isCarbonIndexFile(file.getName()));
- }
- });
-
- //if there are no fact and msr metadata files present then no need to keep
- //entry in metadata.
- if (filesToBeDeleted.length == 0) {
- status = true;
- } else {
-
- for (CarbonFile eachFile : filesToBeDeleted) {
- if (!eachFile.delete()) {
- LOGGER.warn("Unable to delete the file as per delete command " + eachFile
- .getAbsolutePath());
- status = false;
- } else {
- status = true;
- }
- }
- }
- // need to delete the complete folder.
- if (status) {
- if (!file.delete()) {
- LOGGER.warn("Unable to delete the folder as per delete command " + file
- .getAbsolutePath());
- }
- }
-
- } else {
- LOGGER.warn("Files are not found in segment " + path
- + " it seems, files are already being deleted");
- }
-
- }
- } catch (IOException e) {
- LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
- }
- }
- }
- }
-
- private static boolean checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad,
- boolean isForceDelete) {
- if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() ||
- SegmentStatus.COMPACTED == oneLoad.getSegmentStatus() ||
- SegmentStatus.INSERT_IN_PROGRESS == oneLoad.getSegmentStatus() ||
- SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneLoad.getSegmentStatus())
- && oneLoad.getVisibility().equalsIgnoreCase("true")) {
- if (isForceDelete) {
- return true;
- }
- long deletionTime = oneLoad.getModificationOrdeletionTimesStamp();
-
- return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime);
-
- }
-
- return false;
- }
-
- private static boolean checkIfLoadCanBeDeletedPhysically(LoadMetadataDetails oneLoad,
- boolean isForceDelete) {
- if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() ||
- SegmentStatus.COMPACTED == oneLoad.getSegmentStatus())) {
- if (isForceDelete) {
- return true;
- }
- long deletionTime = oneLoad.getModificationOrdeletionTimesStamp();
-
- return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime);
-
- }
-
- return false;
- }
-
- private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId,
- String metadataPath) {
- LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
- for (LoadMetadataDetails oneLoad : currentDetails) {
- if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) {
- return oneLoad;
- }
- }
- return null;
- }
-
- public static boolean deleteLoadFoldersFromFileSystem(
- AbsoluteTableIdentifier absoluteTableIdentifier, boolean isForceDelete,
- LoadMetadataDetails[] details, String metadataPath) {
- boolean isDeleted = false;
- if (details != null && details.length != 0) {
- for (LoadMetadataDetails oneLoad : details) {
- if (checkIfLoadCanBeDeleted(oneLoad, isForceDelete)) {
- ICarbonLock segmentLock = CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier,
- CarbonTablePath.addSegmentPrefix(oneLoad.getLoadName()) + LockUsage.LOCK);
- try {
- if (oneLoad.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
- || oneLoad.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS) {
- if (segmentLock.lockWithRetries(1, 5)) {
- LOGGER.info("Info: Acquired segment lock on segment:" + oneLoad.getLoadName());
- LoadMetadataDetails currentDetails =
- getCurrentLoadStatusOfSegment(oneLoad.getLoadName(), metadataPath);
- if (currentDetails != null && checkIfLoadCanBeDeleted(currentDetails,
- isForceDelete)) {
- oneLoad.setVisibility("false");
- isDeleted = true;
- LOGGER.info("Info: Deleted the load " + oneLoad.getLoadName());
- }
- } else {
- LOGGER.info("Info: Load in progress for segment" + oneLoad.getLoadName());
- return isDeleted;
- }
- } else {
- oneLoad.setVisibility("false");
- isDeleted = true;
- LOGGER.info("Info: Deleted the load " + oneLoad.getLoadName());
- }
- } finally {
- segmentLock.unlock();
- LOGGER.info("Info: Segment lock on segment:" + oneLoad.getLoadName() + " is released");
- }
- }
- }
- }
- return isDeleted;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index 6663683..51d2cf9 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -21,7 +21,7 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-spark-common</artifactId>
+ <artifactId>carbondata-hadoop</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index 51ca09c..e06200a 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -28,6 +28,7 @@ import java.util.Objects;
import org.apache.carbondata.common.Strings;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
@@ -40,7 +41,7 @@ import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.ThriftWriter;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.spark.util.DataLoadingUtil;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModelBuilder;
/**
* Biulder for {@link CarbonWriter}
@@ -94,9 +95,9 @@ public class CarbonWriterBuilder {
}
/**
- * Build a {@link CSVCarbonWriter}, which accepts row in CSV format
+ * Build a {@link CarbonWriter}, which accepts row in CSV format
*/
- public CarbonWriter buildWriterForCSVInput() throws IOException {
+ public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOptionException {
Objects.requireNonNull(schema, "schema should not be null");
Objects.requireNonNull(path, "path should not be null");
@@ -113,7 +114,7 @@ public class CarbonWriterBuilder {
}
/**
- * Build a {@link AvroCarbonWriter}, which accepts Avro object
+ * Build a {@link CarbonWriter}, which accepts Avro object
* @return
* @throws IOException
*/
@@ -184,11 +185,13 @@ public class CarbonWriterBuilder {
/**
* Build a {@link CarbonLoadModel}
*/
- private CarbonLoadModel buildLoadModel(CarbonTable table) {
+ private CarbonLoadModel buildLoadModel(CarbonTable table)
+ throws InvalidLoadOptionException, IOException {
Map<String, String> options = new HashMap<>();
if (sortColumns != null) {
options.put("sort_columns", Strings.mkString(sortColumns, ","));
}
- return DataLoadingUtil.buildCarbonLoadModelJava(table, options);
+ CarbonLoadModelBuilder builder = new CarbonLoadModelBuilder(table);
+ return builder.build(options);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
index 531ec7c..aca2b2d 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
@@ -77,7 +77,7 @@ public class CSVCarbonWriterSuite {
writer.write(new String[]{"robot" + i, String.valueOf(i), String.valueOf((double) i / 2)});
}
writer.close();
- } catch (IOException e) {
+ } catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
index 6316d84..bc7b042 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamSinkFactory.scala
@@ -34,10 +34,9 @@ import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel
+import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
-import org.apache.carbondata.spark.util.DataLoadingUtil
import org.apache.carbondata.streaming.segment.StreamSegment
/**
@@ -209,17 +208,15 @@ object StreamSinkFactory {
segmentId: String): CarbonLoadModel = {
val carbonProperty: CarbonProperties = CarbonProperties.getInstance()
carbonProperty.addProperty("zookeeper.enable.lock", "false")
- val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, parameters)
+ val optionsFinal = LoadOption.fillOptionWithDefaultValue(parameters.asJava)
optionsFinal.put("sort_scope", "no_sort")
if (parameters.get("fileheader").isEmpty) {
optionsFinal.put("fileheader", carbonTable.getCreateOrderColumn(carbonTable.getTableName)
.asScala.map(_.getColName).mkString(","))
}
val carbonLoadModel = new CarbonLoadModel()
- DataLoadingUtil.buildCarbonLoadModel(
- carbonTable,
- carbonProperty,
- parameters,
+ new CarbonLoadModelBuilder(carbonTable).build(
+ parameters.asJava,
optionsFinal,
carbonLoadModel,
hadoopConf)
[45/54] [abbrv] carbondata git commit: [CARBONDATA-2206] Fixed lucene
datamap evaluation issue in executor
Posted by ra...@apache.org.
[CARBONDATA-2206] Fixed lucene datamap evaluation issue in executor
In case of MatchExpression it should return same bitset from RowLevelFilterExecuterImpl
This closes #2010
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5eb476f5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5eb476f5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5eb476f5
Branch: refs/heads/master
Commit: 5eb476f5c27c721ceb6950000fe6f0471a833f27
Parents: bbb1092
Author: ravipesala <ra...@gmail.com>
Authored: Tue Feb 27 19:14:15 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapChooser.java | 3 +-
.../core/scan/expression/MatchExpression.java | 57 ++++++++++++++++++++
.../executer/RowLevelFilterExecuterImpl.java | 4 ++
.../lucene/LuceneFineGrainDataMapSuite.scala | 8 ++-
.../datamap/expression/MatchExpression.java | 56 -------------------
.../spark/sql/optimizer/CarbonFilters.scala | 3 +-
6 files changed, 67 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index c8c971d..f9214a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -228,7 +228,8 @@ public class DataMapChooser {
private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions,
Set<ExpressionType> expressionTypes) {
- if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH)) {
+ if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH) &&
+ expressionTypes.contains(ExpressionType.TEXT_MATCH)) {
// TODO: fix it with right logic
return true;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
new file mode 100644
index 0000000..3677b51
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/MatchExpression.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+@InterfaceAudience.Internal
+public class MatchExpression extends Expression {
+ private String queryString;
+
+ public MatchExpression(String queryString) {
+ this.queryString = queryString;
+ }
+
+ @Override
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return new ExpressionResult(DataTypes.BOOLEAN,true);
+ }
+
+ @Override
+ public ExpressionType getFilterExpressionType() {
+ return ExpressionType.TEXT_MATCH;
+ }
+
+ @Override
+ public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+
+ }
+
+ @Override
+ public String getString() {
+ return queryString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 540607d..2bd49ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -50,6 +50,7 @@ import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.MatchExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
@@ -191,6 +192,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ if (exp instanceof MatchExpression) {
+ return rawBlockletColumnChunks.getBitSetGroup();
+ }
readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 4766281..5e28e8a 100644
--- a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -61,12 +61,10 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- sql("SELECT * FROM datamap_test ORDER BY id").show
-
// sql("select * from normal_test where name='n34000'").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')").show
- sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')").show
+ checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')"), sql(s"select * from datamap_test where name='n10'"))
+// checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')"), sql(s"SELECT * FROM datamap_test WHERE name like 'n10%'"))
+ checkAnswer(sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')"), sql(s"SELECT * FROM datamap_test WHERE city='c020'"))
// checkAnswer(
// sql("select * from datamap_test where match('name:n34000')"),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
deleted file mode 100644
index fceb729..0000000
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.expression;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.ExpressionResult;
-import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-
-@InterfaceAudience.Internal
-public class MatchExpression extends Expression {
- private String queryString;
-
- public MatchExpression(String queryString) {
- this.queryString = queryString;
- }
-
- @Override
- public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return null;
- }
-
- @Override
- public ExpressionType getFilterExpressionType() {
- return ExpressionType.TEXT_MATCH;
- }
-
- @Override
- public void findAndSetChild(Expression oldExpr, Expression newExpr) {
-
- }
-
- @Override
- public String getString() {
- return queryString;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5eb476f5/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 0aedd40..2213fdb 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -37,7 +37,7 @@ import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.SegmentFileStore
import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
@@ -46,7 +46,6 @@ import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.datamap.TextMatch
-import org.apache.carbondata.datamap.expression.MatchExpression
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil
[05/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
new file mode 100644
index 0000000..fde4e55
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/DataBlockIterator.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.BlockletFullScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.util.TaskMetricsMap;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public class DataBlockIterator extends CarbonIterator<List<Object[]>> {
+
+ /**
+ * iterator which will be used to iterate over blocklets
+ */
+ private BlockletIterator blockletIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ private ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ private BlockletScanner blockletScanner;
+
+ /**
+ * batch size of result
+ */
+ private int batchSize;
+
+ private ExecutorService executorService;
+
+ private Future<BlockletScannedResult> future;
+
+ private Future<RawBlockletColumnChunks> futureIo;
+
+ private BlockletScannedResult scannedResult;
+
+ private BlockExecutionInfo blockExecutionInfo;
+
+ private FileReader fileReader;
+
+ private AtomicBoolean nextBlock;
+
+ private AtomicBoolean nextRead;
+
+ public DataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileReader fileReader,
+ int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
+ this.blockExecutionInfo = blockExecutionInfo;
+ this.fileReader = fileReader;
+ blockletIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+ blockExecutionInfo.getNumberOfBlockToScan());
+ if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new BlockletFilterScanner(blockExecutionInfo, queryStatisticsModel);
+ } else {
+ blockletScanner = new BlockletFullScanner(blockExecutionInfo, queryStatisticsModel);
+ }
+ this.scannerResultAggregator =
+ ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
+ this.batchSize = batchSize;
+ this.executorService = executorService;
+ this.nextBlock = new AtomicBoolean(false);
+ this.nextRead = new AtomicBoolean(false);
+ }
+
+ @Override
+ public List<Object[]> next() {
+ List<Object[]> collectedResult = null;
+ if (updateScanner()) {
+ collectedResult = this.scannerResultAggregator.collectResultInRow(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectResultInRow(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
+ }
+ } else {
+ collectedResult = new ArrayList<>();
+ }
+ return collectedResult;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ if (null != scannedResult) {
+ scannedResult.freeMemory();
+ }
+ return blockletIterator.hasNext() || nextBlock.get() || nextRead.get();
+ }
+ }
+
+ /**
+ * Return true if scan result if non-empty
+ */
+ private boolean updateScanner() {
+ try {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ scannedResult = processNextBlocklet();
+ while (scannedResult != null) {
+ if (scannedResult.hasNext()) {
+ return true;
+ }
+ scannedResult = processNextBlocklet();
+ }
+ nextBlock.set(false);
+ nextRead.set(false);
+ return false;
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private BlockletScannedResult processNextBlocklet() throws Exception {
+ BlockletScannedResult result = null;
+ if (blockExecutionInfo.isPrefetchBlocklet()) {
+ if (blockletIterator.hasNext() || nextBlock.get() || nextRead.get()) {
+ if (future == null) {
+ future = scanNextBlockletAsync();
+ }
+ result = future.get();
+ nextBlock.set(false);
+ if (blockletIterator.hasNext() || nextRead.get()) {
+ nextBlock.set(true);
+ future = scanNextBlockletAsync();
+ }
+ }
+ } else {
+ if (blockletIterator.hasNext()) {
+ RawBlockletColumnChunks rawChunks = readNextBlockletColumnChunks();
+ if (rawChunks != null) {
+ result = blockletScanner.scanBlocklet(rawChunks);
+ }
+ }
+ }
+ return result;
+ }
+
+ private RawBlockletColumnChunks readNextBlockletColumnChunks() throws IOException {
+ RawBlockletColumnChunks rawBlockletColumnChunks = getNextBlockletColumnChunks();
+ if (rawBlockletColumnChunks != null) {
+ blockletScanner.readBlocklet(rawBlockletColumnChunks);
+ return rawBlockletColumnChunks;
+ }
+ return null;
+ }
+
+ private RawBlockletColumnChunks getNextBlockletColumnChunks() {
+ RawBlockletColumnChunks rawBlockletColumnChunks = null;
+ do {
+ DataRefNode dataBlock = blockletIterator.next();
+ if (dataBlock.getColumnsMaxValue() == null || blockletScanner.isScanRequired(dataBlock)) {
+ rawBlockletColumnChunks = RawBlockletColumnChunks.newInstance(
+ blockExecutionInfo.getTotalNumberDimensionToRead(),
+ blockExecutionInfo.getTotalNumberOfMeasureToRead(), fileReader, dataBlock);
+ }
+ } while (rawBlockletColumnChunks == null && blockletIterator.hasNext());
+ return rawBlockletColumnChunks;
+ }
+
+ private Future<BlockletScannedResult> scanNextBlockletAsync() {
+ return executorService.submit(new Callable<BlockletScannedResult>() {
+ @Override public BlockletScannedResult call() throws Exception {
+ if (futureIo == null) {
+ futureIo = readNextBlockletAsync();
+ }
+ RawBlockletColumnChunks rawBlockletColumnChunks = futureIo.get();
+ futureIo = null;
+ nextRead.set(false);
+ if (rawBlockletColumnChunks != null) {
+ if (blockletIterator.hasNext()) {
+ nextRead.set(true);
+ futureIo = readNextBlockletAsync();
+ }
+ return blockletScanner.scanBlocklet(rawBlockletColumnChunks);
+ }
+ return null;
+ }
+ });
+ }
+
+ private Future<RawBlockletColumnChunks> readNextBlockletAsync() {
+ return executorService.submit(new Callable<RawBlockletColumnChunks>() {
+ @Override public RawBlockletColumnChunks call() throws Exception {
+ try {
+ TaskMetricsMap.getInstance().registerThreadCallback();
+ if (blockletIterator.hasNext()) {
+ return readNextBlockletColumnChunks();
+ } else {
+ return null;
+ }
+ } finally {
+ // update read bytes metrics for this thread
+ TaskMetricsMap.getInstance().updateReadBytes(Thread.currentThread().getId());
+ }
+ }
+ });
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ if (updateScanner()) {
+ this.scannerResultAggregator.collectResultInColumnarBatch(scannedResult, columnarBatch);
+ }
+ }
+
+
+ /**
+ * Close the resources
+ */
+ public void close() {
+ // free the current scanned result
+ if (null != scannedResult && !scannedResult.hasNext()) {
+ scannedResult.freeMemory();
+ }
+ // free any pre-fetched memory if present
+ if (null != future) {
+ try {
+ BlockletScannedResult blockletScannedResult = future.get();
+ if (blockletScannedResult != null) {
+ blockletScannedResult.freeMemory();
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
new file mode 100644
index 0000000..6b7e880
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/RawBlockletColumnChunks.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * Contains dimension and measure raw column chunks of one blocklet
+ */
+public class RawBlockletColumnChunks {
+
+ /**
+ * dimension column data chunk
+ */
+ private DimensionRawColumnChunk[] dimensionRawColumnChunks;
+
+ /**
+ * measure column data chunk
+ */
+ private MeasureRawColumnChunk[] measureRawColumnChunks;
+
+ /**
+ * file reader which will use to read the block from file
+ */
+ private FileReader fileReader;
+
+ /**
+ * data block
+ */
+ private DataRefNode dataBlock;
+
+ private BitSetGroup bitSetGroup;
+
+ private RawBlockletColumnChunks() { }
+
+ public static RawBlockletColumnChunks newInstance(int numberOfDimensionChunk,
+ int numberOfMeasureChunk, FileReader fileReader, DataRefNode dataBlock) {
+ RawBlockletColumnChunks instance = new RawBlockletColumnChunks();
+ instance.dimensionRawColumnChunks = new DimensionRawColumnChunk[numberOfDimensionChunk];
+ instance.measureRawColumnChunks = new MeasureRawColumnChunk[numberOfMeasureChunk];
+ instance.fileReader = fileReader;
+ instance.dataBlock = dataBlock;
+ return instance;
+ }
+
+ /**
+ * @return the dimensionRawColumnChunks
+ */
+ public DimensionRawColumnChunk[] getDimensionRawColumnChunks() {
+ return dimensionRawColumnChunks;
+ }
+
+ /**
+ * @param dimensionRawColumnChunks the dimensionRawColumnChunks to set
+ */
+ public void setDimensionRawColumnChunks(DimensionRawColumnChunk[] dimensionRawColumnChunks) {
+ this.dimensionRawColumnChunks = dimensionRawColumnChunks;
+ }
+
+ /**
+ * @return the measureRawColumnChunks
+ */
+ public MeasureRawColumnChunk[] getMeasureRawColumnChunks() {
+ return measureRawColumnChunks;
+ }
+
+ /**
+ * @param measureRawColumnChunks the measureRawColumnChunks to set
+ */
+ public void setMeasureRawColumnChunks(MeasureRawColumnChunk[] measureRawColumnChunks) {
+ this.measureRawColumnChunks = measureRawColumnChunks;
+ }
+
+ /**
+ * @return the fileReader
+ */
+ public FileReader getFileReader() {
+ return fileReader;
+ }
+
+ /**
+ * @return the dataBlock
+ */
+ public DataRefNode getDataBlock() {
+ return dataBlock;
+ }
+
+ public BitSetGroup getBitSetGroup() {
+ return bitSetGroup;
+ }
+
+ public void setBitSetGroup(BitSetGroup bitSetGroup) {
+ this.bitSetGroup = bitSetGroup;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
deleted file mode 100644
index 1c97725..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-
-/**
- * Below class will be used to process the block for detail query
- */
-public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
- /**
- * DataBlockIteratorImpl Constructor
- *
- * @param blockExecutionInfo execution information
- */
- public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
- super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel, executorService);
- }
-
- /**
- * It scans the block and returns the result with @batchSize
- *
- * @return Result of @batchSize
- */
- public List<Object[]> next() {
- List<Object[]> collectedResult = null;
- if (updateScanner()) {
- collectedResult = this.scannerResultAggregator.collectData(scannedResult, batchSize);
- while (collectedResult.size() < batchSize && updateScanner()) {
- List<Object[]> data = this.scannerResultAggregator
- .collectData(scannedResult, batchSize - collectedResult.size());
- collectedResult.addAll(data);
- }
- } else {
- collectedResult = new ArrayList<>();
- }
- return collectedResult;
- }
-
- public void processNextBatch(CarbonColumnarBatch columnarBatch) {
- if (updateScanner()) {
- this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
deleted file mode 100644
index b089fad..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ /dev/null
@@ -1,698 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.result;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.mutate.DeleteDeltaVo;
-import org.apache.carbondata.core.mutate.TupleIdEnum;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * Scanned result class which will store and provide the result on request
- */
-public abstract class AbstractScannedResult {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResult.class.getName());
- /**
- * current row number
- */
- protected int currentRow = -1;
-
- protected int pageCounter;
- /**
- * row mapping indexes
- */
- protected int[][] rowMapping;
- /**
- * key size of the fixed length column
- */
- private int fixedLengthKeySize;
- /**
- * total number of rows per page
- */
- private int[] numberOfRows;
-
- /**
- * Total number of rows.
- */
- private int totalNumberOfRows;
- /**
- * to keep track of number of rows process
- */
- protected int rowCounter;
- /**
- * dimension column data chunk
- */
- protected DimensionColumnDataChunk[][] dimensionDataChunks;
-
- /**
- * Raw dimension chunks;
- */
- protected DimensionRawColumnChunk[] dimRawColumnChunks;
-
- /**
- * Raw dimension chunks;
- */
- protected MeasureRawColumnChunk[] msrRawColumnChunks;
- /**
- * measure column data chunk
- */
- protected ColumnPage[][] measureDataChunks;
- /**
- * dictionary column block index in file
- */
- protected int[] dictionaryColumnBlockIndexes;
-
- /**
- * no dictionary column block index in file
- */
- protected int[] noDictionaryColumnBlockIndexes;
-
- /**
- * column group to is key structure info
- * which will be used to get the key from the complete
- * column group key
- * For example if only one dimension of the column group is selected
- * then from complete column group key it will be used to mask the key and
- * get the particular column key
- */
- protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
-
- /**
- *
- */
- private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
-
- private int totalDimensionsSize;
-
- /**
- * blockedId which will be blockId + blocklet number in the block
- */
- private String blockletId;
-
- private long rowId;
-
- /**
- * parent block indexes
- */
- private int[] complexParentBlockIndexes;
-
- /**
- * blockletid+pageumber to deleted reocrd map
- */
- private Map<String, DeleteDeltaVo> deletedRecordMap;
-
- /**
- * current page delete delta vo
- */
- private DeleteDeltaVo currentDeleteDeltaVo;
-
- /**
- * actual blocklet number
- */
- private String blockletNumber;
-
- public AbstractScannedResult(BlockExecutionInfo blockExecutionInfo) {
- this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
- this.noDictionaryColumnBlockIndexes = blockExecutionInfo.getNoDictionaryBlockIndexes();
- this.dictionaryColumnBlockIndexes = blockExecutionInfo.getDictionaryColumnBlockIndex();
- this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
- this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
- this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
- this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
- this.deletedRecordMap = blockExecutionInfo.getDeletedRecordsMap();
- }
-
- /**
- * Below method will be used to set the dimension chunks
- * which will be used to create a row
- *
- * @param dataChunks dimension chunks used in query
- */
- public void setDimensionChunks(DimensionColumnDataChunk[][] dataChunks) {
- this.dimensionDataChunks = dataChunks;
- }
-
- /**
- * Below method will be used to set the measure column chunks
- *
- * @param measureDataChunks measure data chunks
- */
- public void setMeasureChunks(ColumnPage[][] measureDataChunks) {
- this.measureDataChunks = measureDataChunks;
- }
-
- public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
- this.dimRawColumnChunks = dimRawColumnChunks;
- }
-
- public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
- this.msrRawColumnChunks = msrRawColumnChunks;
- }
-
- /**
- * Below method will be used to get the chunk based in measure ordinal
- *
- * @param ordinal measure ordinal
- * @return measure column chunk
- */
- public ColumnPage getMeasureChunk(int ordinal) {
- return measureDataChunks[ordinal][pageCounter];
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected byte[] getDictionaryKeyArray(int rowId) {
- byte[] completeKey = new byte[fixedLengthKeySize];
- int offset = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- offset += dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillChunkData(completeKey, offset, rowId,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * in integer array format which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected int[] getDictionaryKeyIntegerArray(int rowId) {
- int[] completeKey = new int[totalDimensionsSize];
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowId, column, completeKey,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Fill the column data of dictionary to vector
- */
- public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the measure column data to vector
- */
- public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
- for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller
- .fillMeasureVector(measureDataChunks[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
- }
- }
-
- public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
- for (int i = 0; i < vectorInfos.length; i++) {
- int offset = vectorInfos[i].offset;
- int len = offset + vectorInfos[i].size;
- int vectorOffset = vectorInfos[i].vectorOffset;
- CarbonColumnVector vector = vectorInfos[i].vector;
- for (int j = offset; j < len; j++) {
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- vectorInfos[i].genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks,
- rowMapping == null ? j : rowMapping[pageCounter][j], pageCounter, dataOutput);
- Object data = vectorInfos[i].genericQueryType
- .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
- vector.putObject(vectorOffset++, data);
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarImplicitBatch(ColumnVectorInfo[] vectorInfo) {
- for (int i = 0; i < vectorInfo.length; i++) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[i];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int j = offset; j < len; j++) {
- // Considering only String case now as we support only
- String data = getBlockletId();
- if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
- .equals(columnVectorInfo.dimension.getColumnName())) {
- data = data + CarbonCommonConstants.FILE_SEPARATOR + pageCounter
- + CarbonCommonConstants.FILE_SEPARATOR + (rowMapping == null ?
- j :
- rowMapping[pageCounter][j]);
- }
- vector.putBytes(vectorOffset++,
- data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
- }
- }
- }
-
- /**
- * Just increment the counter incase of query only on measures.
- */
- public void incrementCounter() {
- rowCounter++;
- currentRow++;
- }
-
- /**
- * Just increment the page counter and reset the remaining counters.
- */
- public void incrementPageCounter() {
- rowCounter = 0;
- currentRow = -1;
- pageCounter++;
- fillDataChunks();
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
- }
- }
-
- /**
- * This case is used only in case of compaction, since it does not use filter flow.
- */
- public void fillDataChunks() {
- freeDataChunkMemory();
- if (pageCounter >= numberOfRows.length) {
- return;
- }
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (dimensionDataChunks[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
- dimensionDataChunks[i][pageCounter] =
- dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
- }
- }
-
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (measureDataChunks[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
- measureDataChunks[i][pageCounter] =
- msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
- }
- }
- }
-
- // free the memory for the last page chunk
- private void freeDataChunkMemory() {
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (pageCounter > 0 && dimensionDataChunks[i][pageCounter - 1] != null) {
- dimensionDataChunks[i][pageCounter - 1].freeMemory();
- dimensionDataChunks[i][pageCounter - 1] = null;
- }
- }
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (pageCounter > 0 && measureDataChunks[i][pageCounter - 1] != null) {
- measureDataChunks[i][pageCounter - 1].freeMemory();
- measureDataChunks[i][pageCounter - 1] = null;
- }
- }
- }
-
- public int numberOfpages() {
- return numberOfRows.length;
- }
-
- /**
- * Get total rows in the current page
- *
- * @return
- */
- public int getCurrentPageRowCount() {
- return numberOfRows[pageCounter];
- }
-
- public int getCurrentPageCounter() {
- return pageCounter;
- }
-
- /**
- * increment the counter.
- */
- public void setRowCounter(int rowCounter) {
- this.rowCounter = rowCounter;
- }
-
- /**
- * Below method will be used to get the dimension data based on dimension
- * ordinal and index
- *
- * @param dimOrdinal dimension ordinal present in the query
- * @param rowId row index
- * @return dimension data based on row id
- */
- protected byte[] getDimensionData(int dimOrdinal, int rowId) {
- return dimensionDataChunks[dimOrdinal][pageCounter].getChunkData(rowId);
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected byte[][] getNoDictionaryKeyArray(int rowId) {
- byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnBlockIndexes.length][];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] =
- dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId);
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected String[] getNoDictionaryKeyStringArray(int rowId) {
- String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] = new String(
- dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * @return blockletId
- */
- public String getBlockletId() {
- return blockletId;
- }
-
- /**
- * @param blockletId
- */
- public void setBlockletId(String blockletId) {
- this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
- blockletNumber = CarbonUpdateUtil.getRequiredFieldFromTID(blockletId, TupleIdEnum.BLOCKLET_ID);
- // if deleted recors map is present for this block
- // then get the first page deleted vo
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + '_' + pageCounter);
- }
- }
-
- /**
- * @return blockletId
- */
- public long getRowId() {
- return rowId;
- }
-
- /**
- * @param rowId
- */
- public void setRowId(long rowId) {
- this.rowId = rowId;
- }
-
- /**
- * Below method will be used to get the complex type keys array based
- * on row id for all the complex type dimension selected in query
- *
- * @param rowId row number
- * @return complex type key array for all the complex dimension selected in query
- */
- protected byte[][] getComplexTypeKeyArray(int rowId) {
- byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
- for (int i = 0; i < complexTypeData.length; i++) {
- GenericQueryType genericQueryType =
- complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- genericQueryType
- .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
- dataOutput);
- complexTypeData[i] = byteStream.toByteArray();
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- return complexTypeData;
- }
-
- /**
- * @return return the total number of row after scanning
- */
- public int numberOfOutputRows() {
- return this.totalNumberOfRows;
- }
-
- /**
- * to check whether any more row is present in the result
- *
- * @return
- */
- public boolean hasNext() {
- if (pageCounter < numberOfRows.length && rowCounter < this.numberOfRows[pageCounter]) {
- return true;
- } else if (pageCounter < numberOfRows.length) {
- pageCounter++;
- fillDataChunks();
- rowCounter = 0;
- currentRow = -1;
- if (null != deletedRecordMap) {
- currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
- }
- return hasNext();
- }
- return false;
- }
-
- /**
- * Below method will be used to free the occupied memory
- */
- public void freeMemory() {
- // first free the dimension chunks
- if (null != dimensionDataChunks) {
- for (int i = 0; i < dimensionDataChunks.length; i++) {
- if (null != dimensionDataChunks[i]) {
- for (int j = 0; j < dimensionDataChunks[i].length; j++) {
- if (null != dimensionDataChunks[i][j]) {
- dimensionDataChunks[i][j].freeMemory();
- }
- }
- }
- }
- }
- // free the measure data chunks
- if (null != measureDataChunks) {
- for (int i = 0; i < measureDataChunks.length; i++) {
- if (null != measureDataChunks[i]) {
- for (int j = 0; j < measureDataChunks[i].length; j++) {
- if (null != measureDataChunks[i][j]) {
- measureDataChunks[i][j].freeMemory();
- }
- }
- }
- }
- }
- // free the raw chunks
- if (null != dimRawColumnChunks) {
- for (int i = 0; i < dimRawColumnChunks.length; i++) {
- if (null != dimRawColumnChunks[i]) {
- dimRawColumnChunks[i].freeMemory();
- }
- }
- }
- }
-
- /**
- * As this class will be a flyweight object so
- * for one block all the blocklet scanning will use same result object
- * in that case we need to reset the counter to zero so
- * for new result it will give the result from zero
- */
- public void reset() {
- rowCounter = 0;
- currentRow = -1;
- pageCounter = 0;
- }
-
- /**
- * @param numberOfRows set total of number rows valid after scanning
- */
- public void setNumberOfRows(int[] numberOfRows) {
- this.numberOfRows = numberOfRows;
-
- for (int count : numberOfRows) {
- totalNumberOfRows += count;
- }
- }
-
- /**
- * After applying filter it will return the bit set with the valid row indexes
- * so below method will be used to set the row indexes
- *
- * @param indexes
- */
- public void setIndexes(int[][] indexes) {
- this.rowMapping = indexes;
- }
-
- public int getRowCounter() {
- return rowCounter;
- }
-
- /**
- * will return the current valid row id
- *
- * @return valid row id
- */
- public abstract int getCurrentRowId();
-
- /**
- * @return dictionary key array for all the dictionary dimension
- * selected in query
- */
- public abstract byte[] getDictionaryKeyArray();
-
- /**
- * @return dictionary key array for all the dictionary dimension in integer array forat
- * selected in query
- */
- public abstract int[] getDictionaryKeyIntegerArray();
-
- /**
- * Below method will be used to get the complex type key array
- *
- * @return complex type key array
- */
- public abstract byte[][] getComplexTypeKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract byte[][] getNoDictionaryKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array in string array format for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract String[] getNoDictionaryKeyStringArray();
-
- /**
- * Mark the filtered rows in columnar batch. These rows will not be added to vector batches later.
- * @param columnarBatch
- * @param startRow
- * @param size
- * @param vectorOffset
- */
- public int markFilteredRows(CarbonColumnarBatch columnarBatch, int startRow, int size,
- int vectorOffset) {
- int rowsFiltered = 0;
- if (currentDeleteDeltaVo != null) {
- int len = startRow + size;
- for (int i = startRow; i < len; i++) {
- int rowId = rowMapping != null ? rowMapping[pageCounter][i] : i;
- if (currentDeleteDeltaVo.containsRow(rowId)) {
- columnarBatch.markFiltered(vectorOffset);
- rowsFiltered++;
- }
- vectorOffset++;
- }
- }
- return rowsFiltered;
- }
-
- /**
- * Below method will be used to check row got deleted
- *
- * @param rowId
- * @return is present in deleted row
- */
- public boolean containsDeletedRow(int rowId) {
- if (null != currentDeleteDeltaVo) {
- return currentDeleteDeltaVo.containsRow(rowId);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
deleted file mode 100644
index 56ca2ac..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.carbondata.common.CarbonIterator;
-
-/**
- * Below class holds the query result
- */
-public class BatchResult extends CarbonIterator<Object[]> {
-
- /**
- * list of keys
- */
- protected List<Object[]> rows;
-
- /**
- * counter to check whether all the records are processed or not
- */
- protected int counter;
-
- public BatchResult() {
- this.rows = new ArrayList<>();
- }
-
- /**
- * Below method will be used to get the rows
- *
- * @return
- */
- public List<Object[]> getRows() {
- return rows;
- }
-
- /**
- * Below method will be used to get the set the values
- *
- * @param rows
- */
- public void setRows(List<Object[]> rows) {
- this.rows = rows;
- }
-
- /**
- * This method will return one row at a time based on the counter given.
- * @param counter
- * @return
- */
- public Object[] getRawRow(int counter) {
- return rows.get(counter);
- }
-
- /**
- * For getting the total size.
- * @return
- */
- public int getSize() {
- return rows.size();
- }
-
-
- /**
- * Returns {@code true} if the iteration has more elements.
- *
- * @return {@code true} if the iteration has more elements
- */
- @Override public boolean hasNext() {
- return counter < rows.size();
- }
-
- /**
- * Returns the next element in the iteration.
- *
- * @return the next element in the iteration
- */
- @Override public Object[] next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- Object[] row = rows.get(counter);
- counter++;
- return row;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
new file mode 100644
index 0000000..29404b4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -0,0 +1,618 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.mutate.DeleteDeltaVo;
+import org.apache.carbondata.core.mutate.TupleIdEnum;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Scanned result class which will store and provide the result on request
+ */
+public abstract class BlockletScannedResult {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletScannedResult.class.getName());
+ /**
+ * current row number
+ */
+ protected int currentRow = -1;
+
+ protected int pageCounter;
+ /**
+ * matched rowId for each page
+ */
+ protected int[][] pageFilteredRowId;
+ /**
+ * key size of the fixed length column
+ */
+ private int fixedLengthKeySize;
+ /**
+ * total number of filtered rows for each page
+ */
+ private int[] pageFilteredRowCount;
+
+ /**
+ * to keep track of number of rows process
+ */
+ protected int rowCounter;
+ /**
+ * dimension column data chunk
+ */
+ protected DimensionColumnPage[][] dimensionColumnPages;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected DimensionRawColumnChunk[] dimRawColumnChunks;
+
+ /**
+ * Raw dimension chunks;
+ */
+ protected MeasureRawColumnChunk[] msrRawColumnChunks;
+ /**
+ * measure column data chunk
+ */
+ protected ColumnPage[][] measureColumnPages;
+ /**
+ * dictionary column block index in file
+ */
+ protected int[] dictionaryColumnChunkIndexes;
+
+ /**
+ * no dictionary column chunk index in file
+ */
+ protected int[] noDictionaryColumnChunkIndexes;
+
+ /**
+ * column group to is key structure info
+ * which will be used to get the key from the complete
+ * column group key
+ * For example if only one dimension of the column group is selected
+ * then from complete column group key it will be used to mask the key and
+ * get the particular column key
+ */
+ protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
+
+ /**
+ *
+ */
+ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+
+ private int totalDimensionsSize;
+
+ /**
+ * blockedId which will be blockId + blocklet number in the block
+ */
+ private String blockletId;
+
+ /**
+ * parent block indexes
+ */
+ private int[] complexParentBlockIndexes;
+
+ /**
+ * blockletid+pageumber to deleted reocrd map
+ */
+ private Map<String, DeleteDeltaVo> deletedRecordMap;
+
+ /**
+ * current page delete delta vo
+ */
+ private DeleteDeltaVo currentDeleteDeltaVo;
+
+ /**
+ * actual blocklet number
+ */
+ private String blockletNumber;
+
+ public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo) {
+ this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
+ this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
+ this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();
+ this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
+ this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
+ this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getProjectionDimensions().length;
+ this.deletedRecordMap = blockExecutionInfo.getDeletedRecordsMap();
+ }
+
+ /**
+ * Below method will be used to set the dimension chunks
+ * which will be used to create a row
+ *
+ * @param columnPages dimension chunks used in query
+ */
+ public void setDimensionColumnPages(DimensionColumnPage[][] columnPages) {
+ this.dimensionColumnPages = columnPages;
+ }
+
+ /**
+ * Below method will be used to set the measure column chunks
+ *
+ * @param columnPages measure data chunks
+ */
+ public void setMeasureColumnPages(ColumnPage[][] columnPages) {
+ this.measureColumnPages = columnPages;
+ }
+
+ public void setDimRawColumnChunks(DimensionRawColumnChunk[] dimRawColumnChunks) {
+ this.dimRawColumnChunks = dimRawColumnChunks;
+ }
+
+ public void setMsrRawColumnChunks(MeasureRawColumnChunk[] msrRawColumnChunks) {
+ this.msrRawColumnChunks = msrRawColumnChunks;
+ }
+
+ /**
+ * Below method will be used to get the chunk based in measure ordinal
+ *
+ * @param ordinal measure ordinal
+ * @return measure column chunk
+ */
+ public ColumnPage getMeasureChunk(int ordinal) {
+ return measureColumnPages[ordinal][pageCounter];
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected byte[] getDictionaryKeyArray(int rowId) {
+ byte[] completeKey = new byte[fixedLengthKeySize];
+ int offset = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ offset += dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter].fillRawData(
+ rowId, offset, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillSurrogateKey(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Fill the column data of dictionary to vector
+ */
+ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[dictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillVector(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(dictionaryColumnChunkIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
+ column = dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillVector(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(noDictionaryColumnChunkIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the measure column data to vector
+ */
+ public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
+ for (int i = 0; i < measuresOrdinal.length; i++) {
+ vectorInfo[i].measureVectorFiller
+ .fillMeasureVector(measureColumnPages[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
+ }
+ }
+
+ public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
+ for (int i = 0; i < vectorInfos.length; i++) {
+ int offset = vectorInfos[i].offset;
+ int len = offset + vectorInfos[i].size;
+ int vectorOffset = vectorInfos[i].vectorOffset;
+ CarbonColumnVector vector = vectorInfos[i].vector;
+ for (int j = offset; j < len; j++) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(
+ dimRawColumnChunks,
+ pageFilteredRowId == null ? j : pageFilteredRowId[pageCounter][j], pageCounter,
+ dataOutput);
+ Object data = vectorInfos[i].genericQueryType
+ .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
+ vector.putObject(vectorOffset++, data);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarImplicitBatch(ColumnVectorInfo[] vectorInfo) {
+ for (int i = 0; i < vectorInfo.length; i++) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[i];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int j = offset; j < len; j++) {
+ // Considering only String case now as we support only
+ String data = getBlockletId();
+ if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
+ .equals(columnVectorInfo.dimension.getColumnName())) {
+ data = data + CarbonCommonConstants.FILE_SEPARATOR + pageCounter
+ + CarbonCommonConstants.FILE_SEPARATOR + (pageFilteredRowId == null ?
+ j :
+ pageFilteredRowId[pageCounter][j]);
+ }
+ vector.putBytes(vectorOffset++,
+ data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ }
+ }
+ }
+
+ /**
+ * Just increment the counter incase of query only on measures.
+ */
+ public void incrementCounter() {
+ rowCounter++;
+ currentRow++;
+ }
+
+ /**
+ * Just increment the page counter and reset the remaining counters.
+ */
+ public void incrementPageCounter() {
+ rowCounter = 0;
+ currentRow = -1;
+ pageCounter++;
+ fillDataChunks();
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
+ }
+ }
+
+ /**
+ * This case is used only in case of compaction, since it does not use filter flow.
+ */
+ public void fillDataChunks() {
+ freeDataChunkMemory();
+ if (pageCounter >= pageFilteredRowCount.length) {
+ return;
+ }
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (dimensionColumnPages[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
+ dimensionColumnPages[i][pageCounter] =
+ dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
+ }
+ }
+
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (measureColumnPages[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
+ measureColumnPages[i][pageCounter] =
+ msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
+ }
+ }
+ }
+
+ // free the memory for the last page chunk
+ private void freeDataChunkMemory() {
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (pageCounter > 0 && dimensionColumnPages[i][pageCounter - 1] != null) {
+ dimensionColumnPages[i][pageCounter - 1].freeMemory();
+ dimensionColumnPages[i][pageCounter - 1] = null;
+ }
+ }
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (pageCounter > 0 && measureColumnPages[i][pageCounter - 1] != null) {
+ measureColumnPages[i][pageCounter - 1].freeMemory();
+ measureColumnPages[i][pageCounter - 1] = null;
+ }
+ }
+ }
+
+ public int numberOfpages() {
+ return pageFilteredRowCount.length;
+ }
+
+ /**
+ * Get total rows in the current page
+ *
+ * @return
+ */
+ public int getCurrentPageRowCount() {
+ return pageFilteredRowCount[pageCounter];
+ }
+
+ public int getCurrentPageCounter() {
+ return pageCounter;
+ }
+
+ /**
+ * increment the counter.
+ */
+ public void setRowCounter(int rowCounter) {
+ this.rowCounter = rowCounter;
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected byte[][] getNoDictionaryKeyArray(int rowId) {
+ byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnChunkIndexes.length][];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter].getChunkData(rowId);
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * @return blockletId
+ */
+ public String getBlockletId() {
+ return blockletId;
+ }
+
+ /**
+ * Set blocklet id, which looks like
+ * "Part0/Segment_0/part-0-0_batchno0-0-1517155583332.carbondata/0"
+ */
+ public void setBlockletId(String blockletId) {
+ this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+ blockletNumber = CarbonUpdateUtil.getRequiredFieldFromTID(blockletId, TupleIdEnum.BLOCKLET_ID);
+ // if deleted recors map is present for this block
+ // then get the first page deleted vo
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + '_' + pageCounter);
+ }
+ }
+
+ /**
+ * Below method will be used to get the complex type keys array based
+ * on row id for all the complex type dimension selected in query
+ *
+ * @param rowId row number
+ * @return complex type key array for all the complex dimension selected in query
+ */
+ protected byte[][] getComplexTypeKeyArray(int rowId) {
+ byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
+ for (int i = 0; i < complexTypeData.length; i++) {
+ GenericQueryType genericQueryType =
+ complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ genericQueryType
+ .parseBlocksAndReturnComplexColumnByteArray(dimRawColumnChunks, rowId, pageCounter,
+ dataOutput);
+ complexTypeData[i] = byteStream.toByteArray();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ return complexTypeData;
+ }
+
+ /**
+ * to check whether any more row is present in the result
+ *
+ * @return
+ */
+ public boolean hasNext() {
+ if (pageCounter
+ < pageFilteredRowCount.length && rowCounter < this.pageFilteredRowCount[pageCounter]) {
+ return true;
+ } else if (pageCounter < pageFilteredRowCount.length) {
+ pageCounter++;
+ fillDataChunks();
+ rowCounter = 0;
+ currentRow = -1;
+ if (null != deletedRecordMap) {
+ currentDeleteDeltaVo = deletedRecordMap.get(blockletNumber + "_" + pageCounter);
+ }
+ return hasNext();
+ }
+ return false;
+ }
+
+ /**
+ * Below method will be used to free the occupied memory
+ */
+ public void freeMemory() {
+ // first free the dimension chunks
+ if (null != dimensionColumnPages) {
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ if (null != dimensionColumnPages[i]) {
+ for (int j = 0; j < dimensionColumnPages[i].length; j++) {
+ if (null != dimensionColumnPages[i][j]) {
+ dimensionColumnPages[i][j].freeMemory();
+ }
+ }
+ }
+ }
+ }
+ // free the measure data chunks
+ if (null != measureColumnPages) {
+ for (int i = 0; i < measureColumnPages.length; i++) {
+ if (null != measureColumnPages[i]) {
+ for (int j = 0; j < measureColumnPages[i].length; j++) {
+ if (null != measureColumnPages[i][j]) {
+ measureColumnPages[i][j].freeMemory();
+ }
+ }
+ }
+ }
+ }
+ // free the raw chunks
+ if (null != dimRawColumnChunks) {
+ for (int i = 0; i < dimRawColumnChunks.length; i++) {
+ if (null != dimRawColumnChunks[i]) {
+ dimRawColumnChunks[i].freeMemory();
+ }
+ }
+ }
+ }
+
+ /**
+ * @param pageFilteredRowCount set total of number rows valid after scanning
+ */
+ public void setPageFilteredRowCount(int[] pageFilteredRowCount) {
+ this.pageFilteredRowCount = pageFilteredRowCount;
+ }
+
+ /**
+ * After applying filter it will return the bit set with the valid row indexes
+ * so below method will be used to set the row indexes
+ */
+ public void setPageFilteredRowId(int[][] pageFilteredRowId) {
+ this.pageFilteredRowId = pageFilteredRowId;
+ }
+
+ public int getRowCounter() {
+ return rowCounter;
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ public abstract int getCurrentRowId();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension
+ * selected in query
+ */
+ public abstract byte[] getDictionaryKeyArray();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ public abstract byte[][] getComplexTypeKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract byte[][] getNoDictionaryKeyArray();
+
+ /**
+ * Mark the filtered rows in columnar batch. These rows will not be added to vector batches later.
+ * @param columnarBatch
+ * @param startRow
+ * @param size
+ * @param vectorOffset
+ */
+ public int markFilteredRows(CarbonColumnarBatch columnarBatch, int startRow, int size,
+ int vectorOffset) {
+ int rowsFiltered = 0;
+ if (currentDeleteDeltaVo != null) {
+ int len = startRow + size;
+ for (int i = startRow; i < len; i++) {
+ int rowId = pageFilteredRowId != null ? pageFilteredRowId[pageCounter][i] : i;
+ if (currentDeleteDeltaVo.containsRow(rowId)) {
+ columnarBatch.markFiltered(vectorOffset);
+ rowsFiltered++;
+ }
+ vectorOffset++;
+ }
+ }
+ return rowsFiltered;
+ }
+
+ /**
+ * Below method will be used to check row got deleted
+ *
+ * @param rowId
+ * @return is present in deleted row
+ */
+ public boolean containsDeletedRow(int rowId) {
+ if (null != currentDeleteDeltaVo) {
+ return currentDeleteDeltaVo.containsRow(rowId);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java b/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
new file mode 100644
index 0000000..c129161
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/RowBatch.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.carbondata.common.CarbonIterator;
+
+/**
+ * Below class holds the query result
+ */
+public class RowBatch extends CarbonIterator<Object[]> {
+
+ /**
+ * list of keys
+ */
+ protected List<Object[]> rows;
+
+ /**
+ * counter to check whether all the records are processed or not
+ */
+ protected int counter;
+
+ public RowBatch() {
+ this.rows = new ArrayList<>();
+ }
+
+ /**
+ * Below method will be used to get the rows
+ *
+ * @return
+ */
+ public List<Object[]> getRows() {
+ return rows;
+ }
+
+ /**
+ * Below method will be used to get the set the values
+ *
+ * @param rows
+ */
+ public void setRows(List<Object[]> rows) {
+ this.rows = rows;
+ }
+
+ /**
+ * This method will return one row at a time based on the counter given.
+ * @param counter
+ * @return
+ */
+ public Object[] getRawRow(int counter) {
+ return rows.get(counter);
+ }
+
+ /**
+ * For getting the total size.
+ * @return
+ */
+ public int getSize() {
+ return rows.size();
+ }
+
+
+ /**
+ * Returns {@code true} if the iteration has more elements.
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ return counter < rows.size();
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ *
+ * @return the next element in the iteration
+ */
+ @Override public Object[] next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ Object[] row = rows.get(counter);
+ counter++;
+ return row;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
index 8120310..bcc5634 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
@@ -17,7 +17,7 @@
package org.apache.carbondata.core.scan.result.impl;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
* In case of filter query data will be send
* based on filtered row index
*/
-public class FilterQueryScannedResult extends AbstractScannedResult {
+public class FilterQueryScannedResult extends BlockletScannedResult {
public FilterQueryScannedResult(BlockExecutionInfo tableBlockExecutionInfos) {
super(tableBlockExecutionInfos);
@@ -37,7 +37,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
@Override public byte[] getDictionaryKeyArray() {
++currentRow;
- return getDictionaryKeyArray(rowMapping[pageCounter][currentRow]);
+ return getDictionaryKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -46,7 +46,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
@Override public int[] getDictionaryKeyIntegerArray() {
++currentRow;
- return getDictionaryKeyIntegerArray(rowMapping[pageCounter][currentRow]);
+ return getDictionaryKeyIntegerArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -55,7 +55,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return complex type key array
*/
@Override public byte[][] getComplexTypeKeyArray() {
- return getComplexTypeKeyArray(rowMapping[pageCounter][currentRow]);
+ return getComplexTypeKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -65,17 +65,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return no dictionary key array for all the no dictionary dimension
*/
@Override public byte[][] getNoDictionaryKeyArray() {
- return getNoDictionaryKeyArray(rowMapping[pageCounter][currentRow]);
- }
-
- /**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(rowMapping[pageCounter][currentRow]);
+ return getNoDictionaryKeyArray(pageFilteredRowId[pageCounter][currentRow]);
}
/**
@@ -84,7 +74,7 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
* @return valid row id
*/
@Override public int getCurrentRowId() {
- return rowMapping[pageCounter][currentRow];
+ return pageFilteredRowId[pageCounter][currentRow];
}
/**
@@ -92,10 +82,12 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[dictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowMapping[pageCounter], vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ for (int chunkIndex : this.dictionaryColumnChunkIndexes) {
+ column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
+ pageFilteredRowId[pageCounter],
+ vectorInfo,
+ column,
+ columnGroupKeyStructureInfo.get(chunkIndex));
}
}
@@ -104,10 +96,12 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter]
- .fillConvertedChunkData(rowMapping[pageCounter], vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
+ for (int chunkIndex : this.noDictionaryColumnChunkIndexes) {
+ column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
+ pageFilteredRowId[pageCounter],
+ vectorInfo,
+ column,
+ columnGroupKeyStructureInfo.get(chunkIndex));
}
}
@@ -116,8 +110,10 @@ public class FilterQueryScannedResult extends AbstractScannedResult {
*/
public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller.fillMeasureVectorForFilter(rowMapping[pageCounter],
- measureDataChunks[measuresOrdinal[i]][pageCounter], vectorInfo[i]);
+ vectorInfo[i].measureVectorFiller.fillMeasureVector(
+ pageFilteredRowId[pageCounter],
+ measureColumnPages[measuresOrdinal[i]][pageCounter],
+ vectorInfo[i]);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
index 3978f9e..06687c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
@@ -17,14 +17,14 @@
package org.apache.carbondata.core.scan.result.impl;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* Result provide class for non filter query
* In case of no filter query we need to return
* complete data
*/
-public class NonFilterQueryScannedResult extends AbstractScannedResult {
+public class NonFilterQueryScannedResult extends BlockletScannedResult {
public NonFilterQueryScannedResult(BlockExecutionInfo blockExecutionInfo) {
super(blockExecutionInfo);
@@ -68,16 +68,6 @@ public class NonFilterQueryScannedResult extends AbstractScannedResult {
}
/**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(currentRow);
- }
-
- /**
* will return the current valid row id
*
* @return valid row id
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 6172b40..4e628fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -28,18 +28,17 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNode;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DeleteDeltaInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
+import org.apache.carbondata.core.scan.processor.DataBlockIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -63,23 +62,23 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private static final Map<DeleteDeltaInfo, Object> deleteDeltaToLockObjectMap =
new ConcurrentHashMap<>();
- protected ExecutorService execService;
+ private ExecutorService execService;
/**
* execution info of the block
*/
- protected List<BlockExecutionInfo> blockExecutionInfos;
+ private List<BlockExecutionInfo> blockExecutionInfos;
/**
* file reader which will be used to execute the query
*/
- protected FileHolder fileReader;
+ protected FileReader fileReader;
- protected AbstractDataBlockIterator dataBlockIterator;
+ DataBlockIterator dataBlockIterator;
/**
* QueryStatisticsRecorder
*/
- protected QueryStatisticsRecorder recorder;
+ private QueryStatisticsRecorder recorder;
/**
* number of cores which can be used
*/
@@ -89,7 +88,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
*/
private QueryStatisticsModel queryStatisticsModel;
- public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
ExecutorService execService) {
String batchSizeString =
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE);
@@ -107,7 +106,6 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
this.blockExecutionInfos = infos;
this.fileReader = FileFactory.getFileHolder(
FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getTablePath()));
- this.fileReader.setQueryId(queryModel.getQueryId());
this.fileReader.setReadPageByPage(queryModel.isReadPageByPage());
this.execService = execService;
intialiseInfos();
@@ -130,22 +128,21 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
blockInfo.setDeletedRecordsMap(deletedRowsMap);
}
DataRefNode dataRefNode = blockInfo.getDataBlock().getDataRefNode();
- if (dataRefNode instanceof BlockletDataRefNodeWrapper) {
- BlockletDataRefNodeWrapper wrapper = (BlockletDataRefNodeWrapper) dataRefNode;
- blockInfo.setFirstDataBlock(wrapper);
- blockInfo.setNumberOfBlockToScan(wrapper.numberOfNodes());
-
+ if (dataRefNode instanceof BlockletDataRefNode) {
+ BlockletDataRefNode node = (BlockletDataRefNode) dataRefNode;
+ blockInfo.setFirstDataBlock(node);
+ blockInfo.setNumberOfBlockToScan(node.numberOfNodes());
} else {
DataRefNode startDataBlock =
finder.findFirstDataBlock(dataRefNode, blockInfo.getStartKey());
- while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
+ while (startDataBlock.nodeIndex() < blockInfo.getStartBlockletIndex()) {
startDataBlock = startDataBlock.getNextDataRefNode();
}
long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
//if number of block is less than 0 then take end block.
if (numberOfBlockToScan <= 0) {
DataRefNode endDataBlock = finder.findLastDataBlock(dataRefNode, blockInfo.getEndKey());
- numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1;
}
blockInfo.setFirstDataBlock(startDataBlock);
blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
@@ -230,7 +227,8 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- @Override public boolean hasNext() {
+ @Override
+ public boolean hasNext() {
if ((dataBlockIterator != null && dataBlockIterator.hasNext())) {
return true;
} else if (blockExecutionInfos.size() > 0) {
@@ -240,7 +238,7 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- protected void updateDataBlockIterator() {
+ void updateDataBlockIterator() {
if (dataBlockIterator == null || !dataBlockIterator.hasNext()) {
dataBlockIterator = getDataBlockIterator();
while (dataBlockIterator != null && !dataBlockIterator.hasNext()) {
@@ -249,17 +247,17 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
}
}
- private DataBlockIteratorImpl getDataBlockIterator() {
+ private DataBlockIterator getDataBlockIterator() {
if (blockExecutionInfos.size() > 0) {
BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
blockExecutionInfos.remove(executionInfo);
- return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel,
+ return new DataBlockIterator(executionInfo, fileReader, batchSize, queryStatisticsModel,
execService);
}
return null;
}
- protected void initQueryStatiticsModel() {
+ private void initQueryStatiticsModel() {
this.queryStatisticsModel = new QueryStatisticsModel();
this.queryStatisticsModel.setRecorder(recorder);
QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
index 1efac30..1235789 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
@@ -18,7 +18,7 @@
package org.apache.carbondata.core.scan.result.iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
/**
* Iterator over row result
@@ -28,14 +28,14 @@ public class ChunkRowIterator extends CarbonIterator<Object[]> {
/**
* iterator over chunk result
*/
- private CarbonIterator<BatchResult> iterator;
+ private CarbonIterator<RowBatch> iterator;
/**
* currect chunk
*/
- private BatchResult currentchunk;
+ private RowBatch currentchunk;
- public ChunkRowIterator(CarbonIterator<BatchResult> iterator) {
+ public ChunkRowIterator(CarbonIterator<RowBatch> iterator) {
this.iterator = iterator;
if (iterator.hasNext()) {
currentchunk = iterator.next();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index 747f5a9..c073c78 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -21,14 +21,14 @@ import java.util.concurrent.ExecutorService;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.RowBatch;
/**
* In case of detail query we cannot keep all the records in memory so for
* executing that query are returning a iterator over block and every time next
* call will come it will execute the block and return the result
*/
-public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<BatchResult> {
+public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<RowBatch> {
private final Object lock = new Object();
@@ -37,18 +37,18 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
super(infos, queryModel, execService);
}
- @Override public BatchResult next() {
+ @Override public RowBatch next() {
return getBatchResult();
}
- private BatchResult getBatchResult() {
- BatchResult batchResult = new BatchResult();
+ private RowBatch getBatchResult() {
+ RowBatch rowBatch = new RowBatch();
synchronized (lock) {
updateDataBlockIterator();
if (dataBlockIterator != null) {
- batchResult.setRows(dataBlockIterator.next());
+ rowBatch.setRows(dataBlockIterator.next());
}
}
- return batchResult;
+ return rowBatch;
}
}
[52/54] [abbrv] carbondata git commit: [HOTFIX] Add dava doc for
datamap interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
deleted file mode 100644
index 229e5bf..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/FineGrainBlocklet.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.metadata.schema.table.Writable;
-import org.apache.carbondata.core.util.BitSetGroup;
-
-/**
- * FineGrainBlocklet
- */
-public class FineGrainBlocklet extends Blocklet implements Serializable {
-
- private List<Page> pages;
-
- public FineGrainBlocklet(String blockId, String blockletId, List<Page> pages) {
- super(blockId, blockletId);
- this.pages = pages;
- }
-
- // For serialization purpose
- public FineGrainBlocklet() {
-
- }
-
- public List<Page> getPages() {
- return pages;
- }
-
- public static class Page implements Writable,Serializable {
-
- private int pageId;
-
- private int[] rowId;
-
- public BitSet getBitSet() {
- BitSet bitSet =
- new BitSet(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
- for (int row : rowId) {
- bitSet.set(row);
- }
- return bitSet;
- }
-
- @Override public void write(DataOutput out) throws IOException {
- out.writeInt(pageId);
- out.writeInt(rowId.length);
- for (int i = 0; i < rowId.length; i++) {
- out.writeInt(rowId[i]);
- }
- }
-
- @Override public void readFields(DataInput in) throws IOException {
- pageId = in.readInt();
- int length = in.readInt();
- rowId = new int[length];
- for (int i = 0; i < length; i++) {
- rowId[i] = in.readInt();
- }
- }
-
- public void setPageId(int pageId) {
- this.pageId = pageId;
- }
-
- public void setRowId(int[] rowId) {
- this.rowId = rowId;
- }
- }
-
- public BitSetGroup getBitSetGroup(int numberOfPages) {
- BitSetGroup bitSetGroup = new BitSetGroup(numberOfPages);
- for (int i = 0; i < pages.size(); i++) {
- bitSetGroup.setBitSet(pages.get(i).getBitSet(), pages.get(i).pageId);
- }
- return bitSetGroup;
- }
-
- @Override public void write(DataOutput out) throws IOException {
- super.write(out);
- int size = pages.size();
- out.writeInt(size);
- for (Page page : pages) {
- page.write(out);
- }
- }
-
- @Override public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- int size = in.readInt();
- pages = new ArrayList<>(size);
- for (int i = 0; i < size; i++) {
- Page page = new Page();
- page.readFields(in);
- pages.add(page);
- }
- }
-
- @Override public boolean equals(Object o) {
- return super.equals(o);
- }
-
- @Override public int hashCode() {
- return super.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
new file mode 100644
index 0000000..3ca9c5a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -0,0 +1,971 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.BlockMetaInfo;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
+import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.xerial.snappy.Snappy;
+
+/**
+ * Datamap implementation for blocklet.
+ */
+public class BlockletDataMap extends CoarseGrainDataMap implements Cacheable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletDataMap.class.getName());
+
+ private static int KEY_INDEX = 0;
+
+ private static int MIN_VALUES_INDEX = 1;
+
+ private static int MAX_VALUES_INDEX = 2;
+
+ private static int ROW_COUNT_INDEX = 3;
+
+ private static int FILE_PATH_INDEX = 4;
+
+ private static int PAGE_COUNT_INDEX = 5;
+
+ private static int VERSION_INDEX = 6;
+
+ private static int SCHEMA_UPADATED_TIME_INDEX = 7;
+
+ private static int BLOCK_INFO_INDEX = 8;
+
+ private static int BLOCK_FOOTER_OFFSET = 9;
+
+ private static int LOCATIONS = 10;
+
+ private static int BLOCKLET_ID_INDEX = 11;
+
+ private static int BLOCK_LENGTH = 12;
+
+ private static int TASK_MIN_VALUES_INDEX = 0;
+
+ private static int TASK_MAX_VALUES_INDEX = 1;
+
+ private static int SCHEMA = 2;
+
+ private static int INDEX_PATH = 3;
+
+ private static int INDEX_FILE_NAME = 4;
+
+ private static int SEGMENTID = 5;
+
+ private UnsafeMemoryDMStore unsafeMemoryDMStore;
+
+ private UnsafeMemoryDMStore unsafeMemorySummaryDMStore;
+
+ private SegmentProperties segmentProperties;
+
+ private int[] columnCardinality;
+
+ @Override
+ public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
+ long startTime = System.currentTimeMillis();
+ assert (dataMapModel instanceof BlockletDataMapModel);
+ BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ List<DataFileFooter> indexInfo = fileFooterConverter
+ .getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
+ Path path = new Path(blockletDataMapInfo.getFilePath());
+ byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] segmentId =
+ blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ DataMapRowImpl summaryRow = null;
+ byte[] schemaBinary = null;
+ // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
+ // is id assigned to a blocklet within a part file
+ String tempFilePath = null;
+ int relativeBlockletId = 0;
+ for (DataFileFooter fileFooter : indexInfo) {
+ if (segmentProperties == null) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ schemaBinary = convertSchemaToBinary(columnInTable);
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ createSchema(segmentProperties);
+ createSummarySchema(segmentProperties, schemaBinary, filePath, fileName,
+ segmentId);
+ }
+ TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
+ BlockMetaInfo blockMetaInfo =
+ blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
+ // Here it loads info about all blocklets of index
+ // Only add if the file exists physically. There are scenarios which index file exists inside
+ // merge index but related carbondata files are deleted. In that case we first check whether
+ // the file exists physically or not
+ if (blockMetaInfo != null) {
+ if (fileFooter.getBlockletList() == null) {
+ // This is old store scenario, here blocklet information is not available in index file so
+ // load only block info
+ summaryRow =
+ loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
+ blockMetaInfo);
+ } else {
+ // blocklet ID will start from 0 again only when part file path is changed
+ if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
+ tempFilePath = blockInfo.getFilePath();
+ relativeBlockletId = 0;
+ }
+ summaryRow =
+ loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
+ blockMetaInfo, relativeBlockletId);
+ // this is done because relative blocklet id need to be incremented based on the
+ // total number of blocklets
+ relativeBlockletId += fileFooter.getBlockletList().size();
+ }
+ }
+ }
+ if (unsafeMemoryDMStore != null) {
+ unsafeMemoryDMStore.finishWriting();
+ }
+ if (null != unsafeMemorySummaryDMStore) {
+ addTaskSummaryRowToUnsafeMemoryStore(
+ summaryRow,
+ schemaBinary,
+ filePath,
+ fileName,
+ segmentId);
+ unsafeMemorySummaryDMStore.finishWriting();
+ }
+ LOGGER.info(
+ "Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (
+ System.currentTimeMillis() - startTime));
+ }
+
+ private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
+ SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
+ BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ List<BlockletInfo> blockletList = fileFooter.getBlockletList();
+ CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
+ // Add one row to maintain task level min max for segment pruning
+ if (!blockletList.isEmpty() && summaryRow == null) {
+ summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
+ }
+ for (int index = 0; index < blockletList.size(); index++) {
+ DataMapRow row = new DataMapRowImpl(schema);
+ int ordinal = 0;
+ int taskMinMaxOrdinal = 0;
+ BlockletInfo blockletInfo = blockletList.get(index);
+
+ // add start key as index key
+ row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
+
+ BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
+ byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
+ // compute and set task level min values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues,
+ TASK_MIN_VALUES_INDEX, true);
+ ordinal++;
+ taskMinMaxOrdinal++;
+ byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
+ // compute and set task level max values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
+ TASK_MAX_VALUES_INDEX, false);
+ ordinal++;
+
+ row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
+
+ // add file path
+ byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ row.setByteArray(filePathBytes, ordinal++);
+
+ // add pages
+ row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
+
+ // add version number
+ row.setShort(fileFooter.getVersionId().number(), ordinal++);
+
+ // add schema updated time
+ row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
+
+ // add blocklet info
+ byte[] serializedData;
+ try {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(stream);
+ blockletInfo.write(dataOutput);
+ serializedData = stream.toByteArray();
+ row.setByteArray(serializedData, ordinal++);
+ // Add block footer offset, it is used if we need to read footer of block
+ row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
+ ordinal++;
+ // for relative blockelt id i.e blocklet id that belongs to a particular part file
+ row.setShort((short) relativeBlockletId++, ordinal++);
+ // Store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return summaryRow;
+ }
+
+ private void setLocations(String[] locations, DataMapRow row, int ordinal)
+ throws UnsupportedEncodingException {
+ // Add location info
+ String locationStr = StringUtils.join(locations, ',');
+ row.setByteArray(locationStr.getBytes(CarbonCommonConstants.DEFAULT_CHARSET), ordinal);
+ }
+
+ /**
+ * Load information for the block.It is the case can happen only for old stores
+ * where blocklet information is not available in index file. So load only block information
+ * and read blocklet information in executor.
+ */
+ private DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter,
+ SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
+ BlockMetaInfo blockMetaInfo) {
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
+ CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
+ // Add one row to maintain task level min max for segment pruning
+ if (summaryRow == null) {
+ summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
+ }
+ DataMapRow row = new DataMapRowImpl(schema);
+ int ordinal = 0;
+ int taskMinMaxOrdinal = 0;
+ // add start key as index key
+ row.setByteArray(blockletIndex.getBtreeIndex().getStartKey(), ordinal++);
+
+ BlockletMinMaxIndex minMaxIndex = blockletIndex.getMinMaxIndex();
+ byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
+ byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
+ // update min max values in case of old store
+ byte[][] updatedMinValues =
+ CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
+ byte[][] updatedMaxValues =
+ CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMinValues), ordinal);
+ // compute and set task level min values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMinValues,
+ TASK_MIN_VALUES_INDEX, true);
+ ordinal++;
+ taskMinMaxOrdinal++;
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMaxValues), ordinal);
+ // compute and set task level max values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMaxValues,
+ TASK_MAX_VALUES_INDEX, false);
+ ordinal++;
+
+ row.setInt((int)fileFooter.getNumberOfRows(), ordinal++);
+
+ // add file path
+ byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ row.setByteArray(filePathBytes, ordinal++);
+
+ // add pages
+ row.setShort((short) 0, ordinal++);
+
+ // add version number
+ row.setShort(fileFooter.getVersionId().number(), ordinal++);
+
+ // add schema updated time
+ row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
+
+ // add blocklet info
+ row.setByteArray(new byte[0], ordinal++);
+
+ row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
+ try {
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
+ ordinal++;
+ // for relative blocklet id. Value is -1 because in case of old store blocklet info will
+ // not be present in the index file and in that case we will not knwo the total number of
+ // blocklets
+ row.setShort((short) -1, ordinal++);
+
+ // store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return summaryRow;
+ }
+
+ private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId) {
+ // write the task summary info to unsafe memory store
+ if (null != summaryRow) {
+ // Add column schema , it is useful to generate segment properties in executor.
+ // So we no need to read footer again there.
+ if (schemaBinary != null) {
+ summaryRow.setByteArray(schemaBinary, SCHEMA);
+ }
+ summaryRow.setByteArray(filePath, INDEX_PATH);
+ summaryRow.setByteArray(fileName, INDEX_FILE_NAME);
+ summaryRow.setByteArray(segmentId, SEGMENTID);
+ try {
+ unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Fill the measures min values with minimum , this is needed for backward version compatability
+ * as older versions don't store min values for measures
+ */
+ private byte[][] updateMinValues(byte[][] minValues, int[] minMaxLen) {
+ byte[][] updatedValues = minValues;
+ if (minValues.length < minMaxLen.length) {
+ updatedValues = new byte[minMaxLen.length][];
+ System.arraycopy(minValues, 0, updatedValues, 0, minValues.length);
+ List<CarbonMeasure> measures = segmentProperties.getMeasures();
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ for (int i = 0; i < measures.size(); i++) {
+ buffer.rewind();
+ DataType dataType = measures.get(i).getDataType();
+ if (dataType == DataTypes.BYTE) {
+ buffer.putLong(Byte.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.SHORT) {
+ buffer.putLong(Short.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.INT) {
+ buffer.putLong(Integer.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.LONG) {
+ buffer.putLong(Long.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (DataTypes.isDecimal(dataType)) {
+ updatedValues[minValues.length + i] =
+ DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MIN_VALUE));
+ } else {
+ buffer.putDouble(Double.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ }
+ }
+ }
+ return updatedValues;
+ }
+
+ /**
+ * Fill the measures max values with maximum , this is needed for backward version compatability
+ * as older versions don't store max values for measures
+ */
+ private byte[][] updateMaxValues(byte[][] maxValues, int[] minMaxLen) {
+ byte[][] updatedValues = maxValues;
+ if (maxValues.length < minMaxLen.length) {
+ updatedValues = new byte[minMaxLen.length][];
+ System.arraycopy(maxValues, 0, updatedValues, 0, maxValues.length);
+ List<CarbonMeasure> measures = segmentProperties.getMeasures();
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ for (int i = 0; i < measures.size(); i++) {
+ buffer.rewind();
+ DataType dataType = measures.get(i).getDataType();
+ if (dataType == DataTypes.BYTE) {
+ buffer.putLong(Byte.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.SHORT) {
+ buffer.putLong(Short.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.INT) {
+ buffer.putLong(Integer.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.LONG) {
+ buffer.putLong(Long.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (DataTypes.isDecimal(dataType)) {
+ updatedValues[maxValues.length + i] =
+ DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MAX_VALUE));
+ } else {
+ buffer.putDouble(Double.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ }
+ }
+ }
+ return updatedValues;
+ }
+
+ private DataMapRow addMinMax(int[] minMaxLen, CarbonRowSchema carbonRowSchema,
+ byte[][] minValues) {
+ CarbonRowSchema[] minSchemas =
+ ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
+ DataMapRow minRow = new DataMapRowImpl(minSchemas);
+ int minOrdinal = 0;
+ // min value adding
+ for (int i = 0; i < minMaxLen.length; i++) {
+ minRow.setByteArray(minValues[i], minOrdinal++);
+ }
+ return minRow;
+ }
+
+ /**
+ * This method will compute min/max values at task level
+ *
+ * @param taskMinMaxRow
+ * @param minMaxLen
+ * @param carbonRowSchema
+ * @param minMaxValue
+ * @param ordinal
+ * @param isMinValueComparison
+ */
+ private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen,
+ CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal,
+ boolean isMinValueComparison) {
+ DataMapRow row = taskMinMaxRow.getRow(ordinal);
+ byte[][] updatedMinMaxValues = minMaxValue;
+ if (null == row) {
+ CarbonRowSchema[] minSchemas =
+ ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
+ row = new DataMapRowImpl(minSchemas);
+ } else {
+ byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
+ // Compare and update min max values
+ for (int i = 0; i < minMaxLen.length; i++) {
+ int compare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
+ if (isMinValueComparison) {
+ if (compare < 0) {
+ updatedMinMaxValues[i] = existingMinMaxValues[i];
+ }
+ } else if (compare > 0) {
+ updatedMinMaxValues[i] = existingMinMaxValues[i];
+ }
+ }
+ }
+ int minMaxOrdinal = 0;
+ // min/max value adding
+ for (int i = 0; i < minMaxLen.length; i++) {
+ row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
+ }
+ taskMinMaxRow.setRow(row, ordinal);
+ }
+
+ private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
+ List<CarbonRowSchema> indexSchemas = new ArrayList<>();
+
+ // Index key
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+ getMinMaxSchema(segmentProperties, indexSchemas);
+
+ // for number of rows.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
+
+ // for table block path
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for number of pages.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for version number.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for schema updated time.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ //for blocklet info
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for block footer offset.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ // for locations
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for storing block length.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ unsafeMemoryDMStore =
+ new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
+ }
+
+ /**
+ * Creates the schema to store summary information or the information which can be stored only
+ * once per datamap. It stores datamap level max/min of each column and partition information of
+ * datamap
+ * @param segmentProperties
+ * @throws MemoryException
+ */
+ private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId)
+ throws MemoryException {
+ List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
+ getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
+ // for storing column schema
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
+ // for storing file path
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
+ // for storing file name
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
+ // for storing segmentid
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
+ unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
+ taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
+ }
+
+ private void getMinMaxSchema(SegmentProperties segmentProperties,
+ List<CarbonRowSchema> minMaxSchemas) {
+ // Index key
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ // do it 2 times, one for min and one for max.
+ for (int k = 0; k < 2; k++) {
+ CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
+ for (int i = 0; i < minMaxLen.length; i++) {
+ if (minMaxLen[i] <= 0) {
+ mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
+ } else {
+ mapSchemas[i] =
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
+ }
+ }
+ CarbonRowSchema mapSchema =
+ new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
+ mapSchemas);
+ minMaxSchemas.add(mapSchema);
+ }
+ }
+
+ @Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
+ boolean isScanRequired = FilterExpressionProcessor.isScanRequired(
+ filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
+ getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
+ if (isScanRequired) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
+ if (unsafeMemoryDMStore.getRowCount() == 0) {
+ return new ArrayList<>();
+ }
+ List<Blocklet> blocklets = new ArrayList<>();
+ if (filterExp == null) {
+ int rowCount = unsafeMemoryDMStore.getRowCount();
+ for (int i = 0; i < rowCount; i++) {
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
+ blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
+ }
+ } else {
+ // Remove B-tree jump logic as start and end key prepared is not
+ // correct for old store scenarios
+ int startIndex = 0;
+ int endIndex = unsafeMemoryDMStore.getRowCount();
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ while (startIndex < endIndex) {
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
+ int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
+ String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX),
+ CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ boolean isValid =
+ addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX),
+ getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
+ if (isValid) {
+ blocklets.add(createBlocklet(safeRow, blockletId));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<PartitionSpec> partitions) {
+ if (unsafeMemoryDMStore.getRowCount() == 0) {
+ return new ArrayList<>();
+ }
+ // if it has partitioned datamap but there is no partitioned information stored, it means
+ // partitions are dropped so return empty list.
+ if (partitions != null) {
+ // First get the partitions which are stored inside datamap.
+ String[] fileDetails = getFileDetails();
+ // Check the exact match of partition information inside the stored partitions.
+ boolean found = false;
+ Path folderPath = new Path(fileDetails[0]);
+ for (PartitionSpec spec : partitions) {
+ if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ return new ArrayList<>();
+ }
+ }
+ // Prune with filters if the partitions are existed in this datamap
+ return prune(filterExp, segmentProperties);
+ }
+
+ private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
+ boolean needToScan = false;
+ if (spec.getUuid() != null) {
+ String[] split = spec.getUuid().split("_");
+ if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) {
+ needToScan = true;
+ }
+ } else {
+ needToScan = true;
+ }
+ return needToScan;
+ }
+
+ /**
+ * select the blocks based on column min and max value
+ *
+ * @param filterExecuter
+ * @param maxValue
+ * @param minValue
+ * @param filePath
+ * @param blockletId
+ * @return
+ */
+ private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue,
+ byte[][] minValue, String filePath, int blockletId) {
+ BitSet bitSet = null;
+ if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+ String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
+ // this case will come in case of old store where index file does not contain the
+ // blocklet information
+ if (blockletId != -1) {
+ uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
+ }
+ bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+ .isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
+ } else {
+ bitSet = filterExecuter.isScanRequired(maxValue, minValue);
+ }
+ if (!bitSet.isEmpty()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
+ int index = Integer.parseInt(blockletId);
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(index).convertToSafeRow();
+ return createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX));
+ }
+
+ private byte[][] getMinMaxValue(DataMapRow row, int index) {
+ DataMapRow minMaxRow = row.getRow(index);
+ byte[][] minMax = new byte[minMaxRow.getColumnCount()][];
+ for (int i = 0; i < minMax.length; i++) {
+ minMax[i] = minMaxRow.getByteArray(i);
+ }
+ return minMax;
+ }
+
+ private ExtendedBlocklet createBlocklet(DataMapRow row, int blockletId) {
+ ExtendedBlocklet blocklet = new ExtendedBlocklet(
+ new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
+ blockletId + "");
+ BlockletDetailInfo detailInfo = new BlockletDetailInfo();
+ detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
+ detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
+ detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
+ detailInfo.setBlockletId((short) blockletId);
+ detailInfo.setDimLens(columnCardinality);
+ detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
+ byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
+ BlockletInfo blockletInfo = null;
+ try {
+ if (byteArray.length > 0) {
+ blockletInfo = new BlockletInfo();
+ ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
+ DataInputStream inputStream = new DataInputStream(stream);
+ blockletInfo.readFields(inputStream);
+ inputStream.close();
+ }
+ blocklet.setLocation(
+ new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET)
+ .split(","));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ detailInfo.setBlockletInfo(blockletInfo);
+ blocklet.setDetailInfo(detailInfo);
+ detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
+ detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
+ detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
+ return blocklet;
+ }
+
+ private String[] getFileDetails() {
+ try {
+ String[] fileDetails = new String[3];
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ fileDetails[0] =
+ new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ return fileDetails;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Binary search used to get the first tentative index row based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findStartIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ // if key is matched then get the first entry
+ int currentPos = mid;
+ while (currentPos - 1 >= 0
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos - 1)) == 0) {
+ currentPos--;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ // get the leaf child
+ return childNodeIndex;
+ }
+
+ /**
+ * Binary search used to get the last tentative block based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findEndIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ int currentPos = mid;
+ // if key is matched then get the first entry
+ while (currentPos + 1 < unsafeMemoryDMStore.getRowCount()
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos + 1)) == 0) {
+ currentPos++;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ return childNodeIndex;
+ }
+
+ private DataMapRow convertToRow(IndexKey key) {
+ ByteBuffer buffer =
+ ByteBuffer.allocate(key.getDictionaryKeys().length + key.getNoDictionaryKeys().length + 8);
+ buffer.putInt(key.getDictionaryKeys().length);
+ buffer.putInt(key.getNoDictionaryKeys().length);
+ buffer.put(key.getDictionaryKeys());
+ buffer.put(key.getNoDictionaryKeys());
+ DataMapRowImpl dataMapRow = new DataMapRowImpl(unsafeMemoryDMStore.getSchema());
+ dataMapRow.setByteArray(buffer.array(), 0);
+ return dataMapRow;
+ }
+
+ private byte[] getColumnSchemaBinary() {
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ return unsafeRow.getByteArray(SCHEMA);
+ }
+
+ /**
+ * Convert schema to binary
+ */
+ private byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(stream);
+ dataOutput.writeShort(columnSchemas.size());
+ for (ColumnSchema columnSchema : columnSchemas) {
+ if (columnSchema.getColumnReferenceId() == null) {
+ columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
+ }
+ columnSchema.write(dataOutput);
+ }
+ byte[] byteArray = stream.toByteArray();
+ // Compress with snappy to reduce the size of schema
+ return Snappy.rawCompress(byteArray, byteArray.length);
+ }
+
+ @Override
+ public void clear() {
+ if (unsafeMemoryDMStore != null) {
+ unsafeMemoryDMStore.freeMemory();
+ unsafeMemoryDMStore = null;
+ segmentProperties = null;
+ }
+ // clear task min/max unsafe memory
+ if (null != unsafeMemorySummaryDMStore) {
+ unsafeMemorySummaryDMStore.freeMemory();
+ unsafeMemorySummaryDMStore = null;
+ }
+ }
+
+ @Override
+ public long getFileTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public int getAccessCount() {
+ return 0;
+ }
+
+ @Override
+ public long getMemorySize() {
+ long memoryUsed = 0L;
+ if (unsafeMemoryDMStore != null) {
+ memoryUsed += unsafeMemoryDMStore.getMemoryUsed();
+ }
+ if (null != unsafeMemorySummaryDMStore) {
+ memoryUsed += unsafeMemorySummaryDMStore.getMemoryUsed();
+ }
+ return memoryUsed;
+ }
+
+ public SegmentProperties getSegmentProperties() {
+ return segmentProperties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
new file mode 100644
index 0000000..a383288
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMapFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+/**
+ * Table map for blocklet
+ */
+public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
+ implements BlockletDetailsFetcher, SegmentPropertiesFetcher {
+
+ private static final String NAME = "clustered.btree.blocklet";
+
+ public static final DataMapSchema DATA_MAP_SCHEMA =
+ new DataMapSchema(NAME, BlockletDataMapFactory.class.getName());
+
+ private AbsoluteTableIdentifier identifier;
+
+ // segmentId -> list of index file
+ private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+
+ private Cache<TableBlockIndexUniqueIdentifier, CoarseGrainDataMap> cache;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
+ this.identifier = identifier;
+ cache = CacheProvider.getInstance()
+ .createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
+ }
+
+ @Override
+ public DataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ @Override
+ public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException {
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ return cache.getAll(tableBlockIndexUniqueIdentifiers);
+ }
+
+ private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(
+ Segment segment) throws IOException {
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ segmentMap.get(segment.getSegmentNo());
+ if (tableBlockIndexUniqueIdentifiers == null) {
+ tableBlockIndexUniqueIdentifiers = new ArrayList<>();
+ Map<String, String> indexFiles;
+ if (segment.getSegmentFileName() == null) {
+ String path =
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
+ indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ indexFiles = fileStore.getIndexFiles();
+ }
+ for (Map.Entry<String, String> indexFileEntry: indexFiles.entrySet()) {
+ Path indexFile = new Path(indexFileEntry.getKey());
+ tableBlockIndexUniqueIdentifiers.add(
+ new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(),
+ indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
+ }
+ segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
+ }
+ return tableBlockIndexUniqueIdentifiers;
+ }
+
+ /**
+ * Get the blocklet detail information based on blockletid, blockid and segmentid. This method is
+ * exclusively for BlockletDataMapFactory as detail information is only available in this
+ * default datamap.
+ */
+ @Override
+ public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
+ throws IOException {
+ List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
+ // If it is already detailed blocklet then type cast and return same
+ if (blocklets.size() > 0 && blocklets.get(0) instanceof ExtendedBlocklet) {
+ for (Blocklet blocklet : blocklets) {
+ detailedBlocklets.add((ExtendedBlocklet) blocklet);
+ }
+ return detailedBlocklets;
+ }
+ List<TableBlockIndexUniqueIdentifier> identifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ // Retrieve each blocklets detail information from blocklet datamap
+ for (Blocklet blocklet : blocklets) {
+ detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
+ }
+ return detailedBlocklets;
+ }
+
+ @Override
+ public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment)
+ throws IOException {
+ if (blocklet instanceof ExtendedBlocklet) {
+ return (ExtendedBlocklet) blocklet;
+ }
+ List<TableBlockIndexUniqueIdentifier> identifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ return getExtendedBlocklet(identifiers, blocklet);
+ }
+
+ private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
+ Blocklet blocklet) throws IOException {
+ String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
+ for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
+ if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
+ DataMap dataMap = cache.get(identifier);
+ return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
+ }
+ }
+ throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
+ }
+
+
+ @Override
+ public List<DataMapDistributable> toDistributable(Segment segment) {
+ List<DataMapDistributable> distributables = new ArrayList<>();
+ try {
+ CarbonFile[] carbonIndexFiles;
+ if (segment.getSegmentFileName() == null) {
+ carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ Map<String, String> indexFiles = fileStore.getIndexFiles();
+ carbonIndexFiles = new CarbonFile[indexFiles.size()];
+ int i = 0;
+ for (String indexFile : indexFiles.keySet()) {
+ carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
+ }
+ }
+ for (int i = 0; i < carbonIndexFiles.length; i++) {
+ Path path = new Path(carbonIndexFiles[i].getPath());
+
+ FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ LocatedFileStatus fileStatus = iter.next();
+ String[] location = fileStatus.getBlockLocations()[0].getHosts();
+ BlockletDataMapDistributable distributable =
+ new BlockletDataMapDistributable(path.toString());
+ distributable.setLocations(location);
+ distributables.add(distributable);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return distributables;
+ }
+
+ @Override
+ public void fireEvent(Event event) {
+
+ }
+
+ @Override
+ public void clear(Segment segment) {
+ List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
+ if (blockIndexes != null) {
+ for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
+ DataMap dataMap = cache.getIfPresent(blockIndex);
+ if (dataMap != null) {
+ cache.invalidate(blockIndex);
+ dataMap.clear();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void clear() {
+ for (String segmentId : segmentMap.keySet().toArray(new String[segmentMap.size()])) {
+ clear(new Segment(segmentId, null));
+ }
+ }
+
+ @Override
+ public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
+ BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
+ List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
+ Path indexPath = new Path(mapDistributable.getFilePath());
+ String segmentNo = mapDistributable.getSegment().getSegmentNo();
+ if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+ String parent = indexPath.getParent().toString();
+ identifiers
+ .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
+ } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
+ CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
+ String parentPath = carbonFile.getParentFile().getAbsolutePath();
+ List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
+ for (String indexFile : indexFiles) {
+ identifiers.add(
+ new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
+ segmentNo));
+ }
+ }
+ List<CoarseGrainDataMap> dataMaps;
+ try {
+ dataMaps = cache.getAll(identifiers);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return dataMaps;
+ }
+
+ @Override
+ public DataMapMeta getMeta() {
+ // TODO: pass SORT_COLUMNS into this class
+ return null;
+ }
+
+ @Override public SegmentProperties getSegmentProperties(Segment segment) throws IOException {
+ List<CoarseGrainDataMap> dataMaps = getDataMaps(segment);
+ assert (dataMaps.size() > 0);
+ CoarseGrainDataMap coarseGrainDataMap = dataMaps.get(0);
+ assert (coarseGrainDataMap instanceof BlockletDataMap);
+ BlockletDataMap dataMap = (BlockletDataMap) coarseGrainDataMap;
+ return dataMap.getSegmentProperties();
+ }
+
+ @Override public List<Blocklet> getAllBlocklets(Segment segment, List<PartitionSpec> partitions)
+ throws IOException {
+ List<Blocklet> blocklets = new ArrayList<>();
+ List<CoarseGrainDataMap> dataMaps = getDataMaps(segment);
+ for (CoarseGrainDataMap dataMap : dataMaps) {
+ blocklets.addAll(dataMap.prune(null, getSegmentProperties(segment), partitions));
+ }
+ return blocklets;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index 018c3f7..ebeb278 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -22,7 +22,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.indexstore.BlockMetaInfo;
/**
- * It is the model object to keep the information to build or initialize BlockletIndexDataMap.
+ * It is the model object to keep the information to build or initialize BlockletDataMap.
*/
public class BlockletDataMapModel extends DataMapModel {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
index 50862a7..a11ae8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.constants.CarbonVersionConstants;
import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -31,7 +32,6 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.util.BitSetGroup;
[19/54] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad]
Optimization in reading/writing for sort temp row
Posted by ra...@apache.org.
[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
Pick up the no-sort fields in the row and pack them as bytes array and skip parsing them during merge sort to reduce CPU consumption
This closes #1792
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2b41f140
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2b41f140
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2b41f140
Branch: refs/heads/master
Commit: 2b41f140229c1799178313b257f3779908e69010
Parents: 89cfd8e
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Feb 8 14:35:14 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/util/NonDictionaryUtil.java | 67 +--
.../presto/util/CarbonDataStoreCreator.scala | 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../loading/row/IntermediateSortTempRow.java | 117 +++++
.../loading/sort/SortStepRowHandler.java | 466 +++++++++++++++++++
.../loading/sort/SortStepRowUtil.java | 103 ----
.../sort/unsafe/UnsafeCarbonRowPage.java | 331 ++-----------
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 +--
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 ---
.../UnsafeRowComparatorForNormalDims.java | 59 +++
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 ++----
.../merger/UnsafeIntermediateFileMerger.java | 118 +----
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 -
.../sort/sortdata/IntermediateFileMerger.java | 95 +---
.../IntermediateSortTempRowComparator.java | 73 +++
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94 ----
.../sortdata/RowComparatorForNormalDims.java | 62 ---
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +---
.../sort/sortdata/SortTempFileChunkHolder.java | 174 ++-----
.../sort/sortdata/TableFieldStat.java | 176 +++++++
28 files changed, 1186 insertions(+), 1294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index d6ecfbc..fca1244 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,18 +82,26 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required Dimension from obj []
+ * Method to get the required dictionary Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static Integer getDimension(int index, Object[] row) {
-
- Integer[] dimensions = (Integer[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
-
+ public static int getDictDimension(int index, Object[] row) {
+ int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
return dimensions[index];
+ }
+ /**
+ * Method to get the required non-dictionary & complex from 3-parted row
+ * @param index
+ * @param row
+ * @return
+ */
+ public static byte[] getNoDictOrComplex(int index, Object[] row) {
+ byte[][] nonDictArray = (byte[][]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+ return nonDictArray[index];
}
/**
@@ -108,60 +116,11 @@ public class NonDictionaryUtil {
return measures[index];
}
- public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
-
- return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
- }
-
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][] byteBufferArr,
Object[] measureArray) {
-
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
-
- /**
- * This method will extract the single dimension from the complete high card dims byte[].+ *
- * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
- *
- * @param highCardArr
- * @param index
- * @param highCardinalityCount
- * @param outBuffer
- */
- public static void extractSingleHighCardDims(byte[] highCardArr, int index,
- int highCardinalityCount, ByteBuffer outBuffer) {
- ByteBuffer buff = null;
- short secIndex = 0;
- short firstIndex = 0;
- int length;
- // if the requested index is a last one then we need to calculate length
- // based on byte[] length.
- if (index == highCardinalityCount - 1) {
- // need to read 2 bytes(1 short) to determine starting offset and
- // length can be calculated by array length.
- buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 2);
- } else {
- // need to read 4 bytes(2 short) to determine starting offset and
- // length.
- buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 4);
- }
-
- firstIndex = buff.getShort();
- // if it is a last dimension in high card then this will be last
- // offset.so calculate length from total length
- if (index == highCardinalityCount - 1) {
- secIndex = (short) highCardArr.length;
- } else {
- secIndex = buff.getShort();
- }
-
- length = secIndex - firstIndex;
-
- outBuffer.position(firstIndex);
- outBuffer.limit(outBuffer.position() + length);
-
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index e768660..1bc9812 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -383,7 +383,6 @@ object CarbonDataStoreCreator {
.getInstance.createCache(CacheType.REVERSE_DICTIONARY)
for (i <- set.indices) {
- // val dim = getDimension(dims, i).get
val columnIdentifier: ColumnIdentifier =
new ColumnIdentifier(dims.get(i).getColumnId, null, null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 5124247..0422239 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.parser.impl.RowParserImpl
-import org.apache.carbondata.processing.loading.sort.SortStepRowUtil
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler
import org.apache.carbondata.processing.loading.steps.DataWriterProcessorStepImpl
import org.apache.carbondata.processing.sort.sortdata.SortParameters
import org.apache.carbondata.processing.store.{CarbonFactHandler, CarbonFactHandlerFactory}
@@ -206,7 +206,7 @@ object DataLoadProcessorStepOnSpark {
val model: CarbonLoadModel = modelBroadcast.value.getCopyWithTaskNo(index.toString)
val conf = DataLoadProcessBuilder.createConfiguration(model)
val sortParameters = SortParameters.createSortParameters(conf)
- val sortStepRowUtil = new SortStepRowUtil(sortParameters)
+ val sortStepRowHandler = new SortStepRowHandler(sortParameters)
TaskContext.get().addTaskFailureListener { (t: TaskContext, e: Throwable) =>
wrapException(e, model)
}
@@ -216,7 +216,7 @@ object DataLoadProcessorStepOnSpark {
override def next(): CarbonRow = {
val row =
- new CarbonRow(sortStepRowUtil.convertRow(rows.next().getData))
+ new CarbonRow(sortStepRowHandler.convertRawRowTo3Parts(rows.next().getData))
rowCounter.add(1)
row
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
new file mode 100644
index 0000000..8d351cf
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.loading.row;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+/**
+ * During sort procedure, each row will be written to sort temp file in this logic format.
+ * an intermediate sort temp row consists 3 parts:
+ * dictSort, noDictSort, noSortDimsAndMeasures(dictNoSort, noDictNoSort, measure)
+ */
+public class IntermediateSortTempRow {
+ private int[] dictSortDims;
+ private byte[][] noDictSortDims;
+ private byte[] noSortDimsAndMeasures;
+
+ public IntermediateSortTempRow(int[] dictSortDims, byte[][] noDictSortDims,
+ byte[] noSortDimsAndMeasures) {
+ this.dictSortDims = dictSortDims;
+ this.noDictSortDims = noDictSortDims;
+ this.noSortDimsAndMeasures = noSortDimsAndMeasures;
+ }
+
+ public int[] getDictSortDims() {
+ return dictSortDims;
+ }
+
+ public byte[][] getNoDictSortDims() {
+ return noDictSortDims;
+ }
+
+ public byte[] getNoSortDimsAndMeasures() {
+ return noSortDimsAndMeasures;
+ }
+
+ /**
+ * deserialize from bytes array to get the no sort fields
+ * @param outDictNoSort stores the dict & no-sort fields
+ * @param outNoDictNoSort stores the no-dict & no-sort fields, including complex
+ * @param outMeasures stores the measure fields
+ * @param dataTypes data type for the measure
+ */
+ public void unpackNoSortFromBytes(int[] outDictNoSort, byte[][] outNoDictNoSort,
+ Object[] outMeasures, DataType[] dataTypes) {
+ ByteBuffer rowBuffer = ByteBuffer.wrap(noSortDimsAndMeasures);
+ // read dict_no_sort
+ int dictNoSortCnt = outDictNoSort.length;
+ for (int i = 0; i < dictNoSortCnt; i++) {
+ outDictNoSort[i] = rowBuffer.getInt();
+ }
+
+ // read no_dict_no_sort (including complex)
+ int noDictNoSortCnt = outNoDictNoSort.length;
+ for (int i = 0; i < noDictNoSortCnt; i++) {
+ short len = rowBuffer.getShort();
+ byte[] bytes = new byte[len];
+ rowBuffer.get(bytes);
+ outNoDictNoSort[i] = bytes;
+ }
+
+ // read measure
+ int measureCnt = outMeasures.length;
+ DataType tmpDataType;
+ Object tmpContent;
+ for (short idx = 0 ; idx < measureCnt; idx++) {
+ if ((byte) 0 == rowBuffer.get()) {
+ outMeasures[idx] = null;
+ continue;
+ }
+
+ tmpDataType = dataTypes[idx];
+ if (DataTypes.BOOLEAN == tmpDataType) {
+ if ((byte) 1 == rowBuffer.get()) {
+ tmpContent = true;
+ } else {
+ tmpContent = false;
+ }
+ } else if (DataTypes.SHORT == tmpDataType) {
+ tmpContent = rowBuffer.getShort();
+ } else if (DataTypes.INT == tmpDataType) {
+ tmpContent = rowBuffer.getInt();
+ } else if (DataTypes.LONG == tmpDataType) {
+ tmpContent = rowBuffer.getLong();
+ } else if (DataTypes.DOUBLE == tmpDataType) {
+ tmpContent = rowBuffer.getDouble();
+ } else if (DataTypes.isDecimal(tmpDataType)) {
+ short len = rowBuffer.getShort();
+ byte[] decimalBytes = new byte[len];
+ rowBuffer.get(decimalBytes);
+ tmpContent = DataTypeUtil.byteToBigDecimal(decimalBytes);
+ } else {
+ throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
+ }
+ outMeasures[idx] = tmpContent;
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
new file mode 100644
index 0000000..f31a2b9
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
+
+/**
+ * This class is used to convert/write/read row in sort step in carbondata.
+ * It consists the following function:
+ * 1. convert raw row & intermediate sort temp row to 3-parted row
+ * 2. read/write intermediate sort temp row to sort temp file & unsafe memory
+ * 3. write raw row directly to sort temp file & unsafe memory as intermediate sort temp row
+ */
+public class SortStepRowHandler implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private int dictSortDimCnt = 0;
+ private int dictNoSortDimCnt = 0;
+ private int noDictSortDimCnt = 0;
+ private int noDictNoSortDimCnt = 0;
+ private int measureCnt;
+
+ // indices for dict & sort dimension columns
+ private int[] dictSortDimIdx;
+ // indices for dict & no-sort dimension columns
+ private int[] dictNoSortDimIdx;
+ // indices for no-dict & sort dimension columns
+ private int[] noDictSortDimIdx;
+ // indices for no-dict & no-sort dimension columns, including complex columns
+ private int[] noDictNoSortDimIdx;
+ // indices for measure columns
+ private int[] measureIdx;
+
+ private DataType[] dataTypes;
+
+ /**
+ * constructor
+ * @param tableFieldStat table field stat
+ */
+ public SortStepRowHandler(TableFieldStat tableFieldStat) {
+ this.dictSortDimCnt = tableFieldStat.getDictSortDimCnt();
+ this.dictNoSortDimCnt = tableFieldStat.getDictNoSortDimCnt();
+ this.noDictSortDimCnt = tableFieldStat.getNoDictSortDimCnt();
+ this.noDictNoSortDimCnt = tableFieldStat.getNoDictNoSortDimCnt();
+ this.measureCnt = tableFieldStat.getMeasureCnt();
+ this.dictSortDimIdx = tableFieldStat.getDictSortDimIdx();
+ this.dictNoSortDimIdx = tableFieldStat.getDictNoSortDimIdx();
+ this.noDictSortDimIdx = tableFieldStat.getNoDictSortDimIdx();
+ this.noDictNoSortDimIdx = tableFieldStat.getNoDictNoSortDimIdx();
+ this.measureIdx = tableFieldStat.getMeasureIdx();
+ this.dataTypes = tableFieldStat.getMeasureDataType();
+ }
+
+ /**
+ * constructor
+ * @param sortParameters sort parameters
+ */
+ public SortStepRowHandler(SortParameters sortParameters) {
+ this(new TableFieldStat(sortParameters));
+ }
+
+ /**
+ * Convert carbon row from raw format to 3-parted format.
+ * This method is used in global-sort.
+ *
+ * @param row raw row whose length is the same as field number
+ * @return 3-parted row whose length is 3. (1 for dict dims ,1 for non-dict and complex,
+ * 1 for measures)
+ */
+ public Object[] convertRawRowTo3Parts(Object[] row) {
+ Object[] holder = new Object[3];
+ try {
+ int[] dictDims
+ = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
+ byte[][] nonDictArray = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
+ Object[] measures = new Object[this.measureCnt];
+
+ // convert dict & data
+ int idxAcc = 0;
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ dictDims[idxAcc++] = (int) row[this.dictSortDimIdx[idx]];
+ }
+
+ // convert dict & no-sort
+ for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
+ dictDims[idxAcc++] = (int) row[this.dictNoSortDimIdx[idx]];
+ }
+ // convert no-dict & sort
+ idxAcc = 0;
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ nonDictArray[idxAcc++] = (byte[]) row[this.noDictSortDimIdx[idx]];
+ }
+ // convert no-dict & no-sort
+ for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
+ nonDictArray[idxAcc++] = (byte[]) row[this.noDictNoSortDimIdx[idx]];
+ }
+
+ // convert measure data
+ for (int idx = 0; idx < this.measureCnt; idx++) {
+ measures[idx] = row[this.measureIdx[idx]];
+ }
+
+ NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
+ } catch (Exception e) {
+ throw new RuntimeException("Problem while converting row to 3 parts", e);
+ }
+ return holder;
+ }
+
+ /**
+ * Convert intermediate sort temp row to 3-parted row.
+ * This method is used in the final merge sort to feed rows to the next write step.
+ *
+ * @param sortTempRow intermediate sort temp row
+ * @return 3-parted row
+ */
+ public Object[] convertIntermediateSortTempRowTo3Parted(IntermediateSortTempRow sortTempRow) {
+ int[] dictDims
+ = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
+ byte[][] noDictArray
+ = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
+
+ int[] dictNoSortDims = new int[this.dictNoSortDimCnt];
+ byte[][] noDictNoSortDims = new byte[this.noDictNoSortDimCnt][];
+ Object[] measures = new Object[this.measureCnt];
+
+ sortTempRow.unpackNoSortFromBytes(dictNoSortDims, noDictNoSortDims, measures, this.dataTypes);
+
+ // dict dims
+ System.arraycopy(sortTempRow.getDictSortDims(), 0 , dictDims,
+ 0, this.dictSortDimCnt);
+ System.arraycopy(dictNoSortDims, 0, dictDims,
+ this.dictSortDimCnt, this.dictNoSortDimCnt);;
+
+ // no dict dims, including complex
+ System.arraycopy(sortTempRow.getNoDictSortDims(), 0,
+ noDictArray, 0, this.noDictSortDimCnt);
+ System.arraycopy(noDictNoSortDims, 0, noDictArray,
+ this.noDictSortDimCnt, this.noDictNoSortDimCnt);
+
+ // measures are already here
+
+ Object[] holder = new Object[3];
+ NonDictionaryUtil.prepareOutObj(holder, dictDims, noDictArray, measures);
+ return holder;
+ }
+
+ /**
+ * Read intermediate sort temp row from InputStream.
+ * This method is used during the merge sort phase to read row from sort temp file.
+ *
+ * @param inputStream input stream
+ * @return a row that contains three parts
+ * @throws IOException if error occrus while reading from stream
+ */
+ public IntermediateSortTempRow readIntermediateSortTempRowFromInputStream(
+ DataInputStream inputStream) throws IOException {
+ int[] dictSortDims = new int[this.dictSortDimCnt];
+ byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
+
+ // read dict & sort dim data
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ dictSortDims[idx] = inputStream.readInt();
+ }
+
+ // read no-dict & sort data
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ short len = inputStream.readShort();
+ byte[] bytes = new byte[len];
+ inputStream.readFully(bytes);
+ noDictSortDims[idx] = bytes;
+ }
+
+ // read no-dict dims & measures
+ int len = inputStream.readInt();
+ byte[] noSortDimsAndMeasures = new byte[len];
+ inputStream.readFully(noSortDimsAndMeasures);
+
+ return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write intermediate sort temp row to OutputStream
+ * This method is used during the merge sort phase to write row to sort temp file.
+ *
+ * @param sortTempRow intermediate sort temp row
+ * @param outputStream output stream
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeIntermediateSortTempRowToOutputStream(IntermediateSortTempRow sortTempRow,
+ DataOutputStream outputStream) throws IOException {
+ // write dict & sort dim
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ outputStream.writeInt(sortTempRow.getDictSortDims()[idx]);
+ }
+
+ // write no-dict & sort dim
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = sortTempRow.getNoDictSortDims()[idx];
+ outputStream.writeShort(bytes.length);
+ outputStream.write(bytes);
+ }
+
+ // write packed no-sort dim & measure
+ outputStream.writeInt(sortTempRow.getNoSortDimsAndMeasures().length);
+ outputStream.write(sortTempRow.getNoSortDimsAndMeasures());
+ }
+
+ /**
+ * Write raw row as an intermediate sort temp row to sort temp file.
+ * This method is used in the beginning of the sort phase. Comparing with converting raw row to
+ * intermediate sort temp row and then writing the converted one, Writing raw row directly will
+ * save the intermediate trivial loss.
+ * This method use an array backend buffer to save memory allocation. The buffer will be reused
+ * for all rows (per thread).
+ *
+ * @param row raw row
+ * @param outputStream output stream
+ * @param rowBuffer array backend buffer
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeRawRowAsIntermediateSortTempRowToOutputStream(Object[] row,
+ DataOutputStream outputStream, ByteBuffer rowBuffer) throws IOException {
+ // write dict & sort
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ outputStream.writeInt((int) row[this.dictSortDimIdx[idx]]);
+ }
+
+ // write no-dict & sort
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
+ outputStream.writeShort(bytes.length);
+ outputStream.write(bytes);
+ }
+
+ // pack no-sort
+ rowBuffer.clear();
+ packNoSortFieldsToBytes(row, rowBuffer);
+ rowBuffer.flip();
+ int packSize = rowBuffer.limit();
+
+ // write no-sort
+ outputStream.writeInt(packSize);
+ outputStream.write(rowBuffer.array(), 0, packSize);
+ }
+
+ /**
+ * Read intermediate sort temp row from unsafe memory.
+ * This method is used during merge sort phase for off-heap sort.
+ *
+ * @param baseObject base object of memory block
+ * @param address address of the row
+ * @return intermediate sort temp row
+ */
+ public IntermediateSortTempRow readIntermediateSortTempRowFromUnsafeMemory(Object baseObject,
+ long address) {
+ int size = 0;
+
+ int[] dictSortDims = new int[this.dictSortDimCnt];
+ byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
+
+ // read dict & sort dim
+ for (int idx = 0; idx < dictSortDims.length; idx++) {
+ dictSortDims[idx] = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ }
+
+ // read no-dict & sort dim
+ for (int idx = 0; idx < noDictSortDims.length; idx++) {
+ short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ byte[] bytes = new byte[length];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+ size += length;
+ noDictSortDims[idx] = bytes;
+ }
+
+ // read no-sort dims & measures
+ int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ byte[] noSortDimsAndMeasures = new byte[len];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
+
+ return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write intermediate sort temp row directly from unsafe memory to stream.
+ * This method is used at the late beginning of the sort phase to write in-memory pages
+ * to sort temp file. Comparing with reading intermediate sort temp row from memory and then
+ * writing it, Writing directly from memory to stream will save the intermediate trivial loss.
+ *
+ * @param baseObject base object of the memory block
+ * @param address base address of the row
+ * @param outputStream output stream
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeIntermediateSortTempRowFromUnsafeMemoryToStream(Object baseObject,
+ long address, DataOutputStream outputStream) throws IOException {
+ int size = 0;
+
+ // dict & sort
+ for (int idx = 0; idx < dictSortDimCnt; idx++) {
+ outputStream.writeInt(CarbonUnsafe.getUnsafe().getInt(baseObject, address + size));
+ size += 4;
+ }
+
+ // no-dict & sort
+ for (int idx = 0; idx < noDictSortDimCnt; idx++) {
+ short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ byte[] bytes = new byte[length];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+ size += length;
+
+ outputStream.writeShort(length);
+ outputStream.write(bytes);
+ }
+
+ // packed no-sort & measure
+ int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ byte[] noSortDimsAndMeasures = new byte[len];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
+ size += len;
+
+ outputStream.writeInt(len);
+ outputStream.write(noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write raw row as an intermediate sort temp row to memory.
+ * This method is used in the beginning of the off-heap sort phase. Comparing with converting
+ * raw row to intermediate sort temp row and then writing the converted one,
+ * Writing raw row directly will save the intermediate trivial loss.
+ * This method use an array backend buffer to save memory allocation. The buffer will be reused
+ * for all rows (per thread).
+ *
+ * @param row raw row
+ * @param baseObject base object of the memory block
+ * @param address base address for the row
+ * @param rowBuffer array backend buffer
+ * @return number of bytes written to memory
+ */
+ public int writeRawRowAsIntermediateSortTempRowToUnsafeMemory(Object[] row,
+ Object baseObject, long address, ByteBuffer rowBuffer) {
+ int size = 0;
+ // write dict & sort
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ CarbonUnsafe.getUnsafe()
+ .putInt(baseObject, address + size, (int) row[this.dictSortDimIdx[idx]]);
+ size += 4;
+ }
+
+ // write no-dict & sort
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) bytes.length);
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
+ bytes.length);
+ size += bytes.length;
+ }
+
+ // convert pack no-sort
+ rowBuffer.clear();
+ packNoSortFieldsToBytes(row, rowBuffer);
+ rowBuffer.flip();
+ int packSize = rowBuffer.limit();
+
+ // write no-sort
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, packSize);
+ size += 4;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(rowBuffer.array(), CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
+ packSize);
+ size += packSize;
+ return size;
+ }
+
+ /**
+ * Pack to no-sort fields to byte array
+ *
+ * @param row raw row
+ * @param rowBuffer byte array backend buffer
+ */
+ private void packNoSortFieldsToBytes(Object[] row, ByteBuffer rowBuffer) {
+ // convert dict & no-sort
+ for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
+ rowBuffer.putInt((int) row[this.dictNoSortDimIdx[idx]]);
+ }
+ // convert no-dict & no-sort
+ for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictNoSortDimIdx[idx]];
+ rowBuffer.putShort((short) bytes.length);
+ rowBuffer.put(bytes);
+ }
+
+ // convert measure
+ Object tmpValue;
+ DataType tmpDataType;
+ for (int idx = 0; idx < this.measureCnt; idx++) {
+ tmpValue = row[this.measureIdx[idx]];
+ tmpDataType = this.dataTypes[idx];
+ if (null == tmpValue) {
+ rowBuffer.put((byte) 0);
+ continue;
+ }
+ rowBuffer.put((byte) 1);
+ if (DataTypes.BOOLEAN == tmpDataType) {
+ if ((boolean) tmpValue) {
+ rowBuffer.put((byte) 1);
+ } else {
+ rowBuffer.put((byte) 0);
+ }
+ } else if (DataTypes.SHORT == tmpDataType) {
+ rowBuffer.putShort((Short) tmpValue);
+ } else if (DataTypes.INT == tmpDataType) {
+ rowBuffer.putInt((Integer) tmpValue);
+ } else if (DataTypes.LONG == tmpDataType) {
+ rowBuffer.putLong((Long) tmpValue);
+ } else if (DataTypes.DOUBLE == tmpDataType) {
+ rowBuffer.putDouble((Double) tmpValue);
+ } else if (DataTypes.isDecimal(tmpDataType)) {
+ byte[] decimalBytes = DataTypeUtil.bigDecimalToByte((BigDecimal) tmpValue);
+ rowBuffer.putShort((short) decimalBytes.length);
+ rowBuffer.put(decimalBytes);
+ } else {
+ throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
deleted file mode 100644
index c4e4756..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort;
-
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-
-public class SortStepRowUtil {
- private int measureCount;
- private int dimensionCount;
- private int complexDimensionCount;
- private int noDictionaryCount;
- private int[] dictDimIdx;
- private int[] nonDictIdx;
- private int[] measureIdx;
-
- public SortStepRowUtil(SortParameters parameters) {
- this.measureCount = parameters.getMeasureColCount();
- this.dimensionCount = parameters.getDimColCount();
- this.complexDimensionCount = parameters.getComplexDimColCount();
- this.noDictionaryCount = parameters.getNoDictionaryCount();
- boolean[] isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
-
- int index = 0;
- int nonDicIndex = 0;
- int allCount = 0;
-
- // be careful that the default value is 0
- this.dictDimIdx = new int[dimensionCount - noDictionaryCount];
- this.nonDictIdx = new int[noDictionaryCount + complexDimensionCount];
- this.measureIdx = new int[measureCount];
-
- // indices for dict dim columns
- for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
- if (isNoDictionaryDimensionColumn[i]) {
- nonDictIdx[nonDicIndex++] = i;
- } else {
- dictDimIdx[index++] = allCount;
- }
- allCount++;
- }
-
- // indices for non dict dim/complex columns
- for (int i = 0; i < complexDimensionCount; i++) {
- nonDictIdx[nonDicIndex++] = allCount;
- allCount++;
- }
-
- // indices for measure columns
- for (int i = 0; i < measureCount; i++) {
- measureIdx[i] = allCount;
- allCount++;
- }
- }
-
- public Object[] convertRow(Object[] data) {
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
- Object[] holder = new Object[3];
- try {
-
- int[] dictDims = new int[dimensionCount - noDictionaryCount];
- byte[][] nonDictArray = new byte[noDictionaryCount + complexDimensionCount][];
- Object[] measures = new Object[measureCount];
-
- // write dict dim data
- for (int idx = 0; idx < dictDimIdx.length; idx++) {
- dictDims[idx] = (int) data[dictDimIdx[idx]];
- }
-
- // write non dict dim data
- for (int idx = 0; idx < nonDictIdx.length; idx++) {
- nonDictArray[idx] = (byte[]) data[nonDictIdx[idx]];
- }
-
- // write measure data
- for (int idx = 0; idx < measureIdx.length; idx++) {
- measures[idx] = data[measureIdx[idx]];
- }
- NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
-
- // increment number if record read
- } catch (Exception e) {
- throw new RuntimeException("Problem while converting row ", e);
- }
- //return out row
- return holder;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
index e5583c2..7ea5cb3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
@@ -19,35 +19,20 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.memory.IntPointerBuffer;
import org.apache.carbondata.core.memory.MemoryBlock;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.memory.UnsafeSortMemoryManager;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
/**
* It can keep the data of prescribed size data in offheap/onheap memory and returns it when needed
*/
public class UnsafeCarbonRowPage {
-
- private boolean[] noDictionaryDimensionMapping;
-
- private boolean[] noDictionarySortColumnMapping;
-
- private int dimensionSize;
-
- private int measureSize;
-
- private DataType[] measureDataType;
-
- private long[] nullSetWords;
-
private IntPointerBuffer buffer;
private int lastSize;
@@ -62,16 +47,14 @@ public class UnsafeCarbonRowPage {
private long taskId;
- public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
- boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, DataType[] type,
- MemoryBlock memoryBlock, boolean saveToDisk, long taskId) {
- this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
- this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
- this.dimensionSize = dimensionSize;
- this.measureSize = measureSize;
- this.measureDataType = type;
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+
+ public UnsafeCarbonRowPage(TableFieldStat tableFieldStat, MemoryBlock memoryBlock,
+ boolean saveToDisk, long taskId) {
+ this.tableFieldStat = tableFieldStat;
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.saveToDisk = saveToDisk;
- this.nullSetWords = new long[((measureSize - 1) >> 6) + 1];
this.taskId = taskId;
buffer = new IntPointerBuffer(this.taskId);
this.dataBlock = memoryBlock;
@@ -80,255 +63,44 @@ public class UnsafeCarbonRowPage {
this.managerType = MemoryManagerType.UNSAFE_MEMORY_MANAGER;
}
- public int addRow(Object[] row) {
- int size = addRow(row, dataBlock.getBaseOffset() + lastSize);
+ public int addRow(Object[] row, ByteBuffer rowBuffer) {
+ int size = addRow(row, dataBlock.getBaseOffset() + lastSize, rowBuffer);
buffer.set(lastSize);
lastSize = lastSize + size;
return size;
}
- private int addRow(Object[] row, long address) {
- if (row == null) {
- throw new RuntimeException("Row is null ??");
- }
- int dimCount = 0;
- int size = 0;
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- } else {
- int value = (int) row[dimCount];
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, value);
- size += 4;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- }
- Arrays.fill(nullSetWords, 0);
- int nullSetSize = nullSetWords.length * 8;
- int nullWordLoc = size;
- size += nullSetSize;
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- Object value = row[mesCount + dimensionSize];
- if (null != value) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- Boolean bval = (Boolean) value;
- CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, bval);
- size += 1;
- } else if (dataType == DataTypes.SHORT) {
- Short sval = (Short) value;
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
- size += 2;
- } else if (dataType == DataTypes.INT) {
- Integer ival = (Integer) value;
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival);
- size += 4;
- } else if (dataType == DataTypes.LONG) {
- Long val = (Long) value;
- CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val);
- size += 8;
- } else if (dataType == DataTypes.DOUBLE) {
- Double doubleVal = (Double) value;
- CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal);
- size += 8;
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal decimalVal = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) bigDecimalInBytes.length);
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- set(nullSetWords, mesCount);
- } else {
- unset(nullSetWords, mesCount);
- }
- }
- CarbonUnsafe.getUnsafe().copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
- address + nullWordLoc, nullSetSize);
- return size;
+ /**
+ * add raw row as intermidiate sort temp row to page
+ *
+ * @param row
+ * @param address
+ * @return
+ */
+ private int addRow(Object[] row, long address, ByteBuffer rowBuffer) {
+ return sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToUnsafeMemory(row,
+ dataBlock.getBaseObject(), address, rowBuffer);
}
- public Object[] getRow(long address, Object[] rowToFill) {
- int dimCount = 0;
- int size = 0;
-
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- col.length);
- size += col.length;
- rowToFill[dimCount] = col;
- } else {
- int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- rowToFill[dimCount] = anInt;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
- size += col.length;
- rowToFill[dimCount] = col;
- }
-
- int nullSetSize = nullSetWords.length * 8;
- Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
- nullSetSize);
- size += nullSetSize;
-
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- if (isSet(nullSetWords, mesCount)) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- Boolean bval = CarbonUnsafe.getUnsafe().getBoolean(baseObject, address + size);
- size += 1;
- rowToFill[dimensionSize + mesCount] = bval;
- } else if (dataType == DataTypes.SHORT) {
- Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- rowToFill[dimensionSize + mesCount] = sval;
- } else if (dataType == DataTypes.INT) {
- Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- rowToFill[dimensionSize + mesCount] = ival;
- } else if (dataType == DataTypes.LONG) {
- Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = val;
- } else if (dataType == DataTypes.DOUBLE) {
- Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = doubleVal;
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- } else {
- rowToFill[dimensionSize + mesCount] = null;
- }
- }
- return rowToFill;
+ /**
+ * get one row from memory address
+ * @param address address
+ * @return one row
+ */
+ public IntermediateSortTempRow getRow(long address) {
+ return sortStepRowHandler.readIntermediateSortTempRowFromUnsafeMemory(
+ dataBlock.getBaseObject(), address);
}
- public void fillRow(long address, DataOutputStream stream) throws IOException {
- int dimCount = 0;
- int size = 0;
-
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- col.length);
- size += col.length;
- stream.writeShort(aShort);
- stream.write(col);
- } else {
- int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- stream.writeInt(anInt);
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
- size += col.length;
- stream.writeShort(aShort);
- stream.write(col);
- }
-
- int nullSetSize = nullSetWords.length * 8;
- Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
- nullSetSize);
- size += nullSetSize;
- for (int i = 0; i < nullSetWords.length; i++) {
- stream.writeLong(nullSetWords[i]);
- }
-
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- if (isSet(nullSetWords, mesCount)) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.SHORT) {
- short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- stream.writeShort(sval);
- } else if (dataType == DataTypes.INT) {
- int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- stream.writeInt(ival);
- } else if (dataType == DataTypes.LONG) {
- long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
- size += 8;
- stream.writeLong(val);
- } else if (dataType == DataTypes.DOUBLE) {
- double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
- size += 8;
- stream.writeDouble(doubleVal);
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- stream.writeShort(aShort);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- }
- }
+ /**
+ * write a row to stream
+ * @param address address of a row
+ * @param stream stream
+ * @throws IOException
+ */
+ public void writeRow(long address, DataOutputStream stream) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowFromUnsafeMemoryToStream(
+ dataBlock.getBaseObject(), address, stream);
}
public void freeMemory() {
@@ -362,27 +134,8 @@ public class UnsafeCarbonRowPage {
return dataBlock;
}
- public static void set(long[] words, int index) {
- int wordOffset = (index >> 6);
- words[wordOffset] |= (1L << index);
- }
-
- public static void unset(long[] words, int index) {
- int wordOffset = (index >> 6);
- words[wordOffset] &= ~(1L << index);
- }
-
- public static boolean isSet(long[] words, int index) {
- int wordOffset = (index >> 6);
- return ((words[wordOffset] & (1L << index)) != 0);
- }
-
- public boolean[] getNoDictionaryDimensionMapping() {
- return noDictionaryDimensionMapping;
- }
-
- public boolean[] getNoDictionarySortColumnMapping() {
- return noDictionarySortColumnMapping;
+ public TableFieldStat getTableFieldStat() {
+ return tableFieldStat;
}
public void setNewDataBlock(MemoryBlock newMemoryBlock) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index 4dd5e44..5d038d3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -41,13 +42,14 @@ import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparator;
-import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDIms;
+import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDims;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.UnsafeIntSortDataFormat;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class UnsafeSortDataRows {
@@ -69,7 +71,8 @@ public class UnsafeSortDataRows {
*/
private SortParameters parameters;
-
+ private TableFieldStat tableFieldStat;
+ private ThreadLocal<ByteBuffer> rowBuffer;
private UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger;
private UnsafeCarbonRowPage rowPage;
@@ -94,7 +97,13 @@ public class UnsafeSortDataRows {
public UnsafeSortDataRows(SortParameters parameters,
UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger, int inMemoryChunkSize) {
this.parameters = parameters;
-
+ this.tableFieldStat = new TableFieldStat(parameters);
+ this.rowBuffer = new ThreadLocal<ByteBuffer>() {
+ @Override protected ByteBuffer initialValue() {
+ byte[] backedArray = new byte[2 * 1024 * 1024];
+ return ByteBuffer.wrap(backedArray);
+ }
+ };
this.unsafeInMemoryIntermediateFileMerger = unsafeInMemoryIntermediateFileMerger;
// observer of writing file in thread
@@ -127,11 +136,7 @@ public class UnsafeSortDataRows {
if (isMemoryAvailable) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size());
}
- this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(), parameters.getMeasureDataType(), baseBlock,
- !isMemoryAvailable, taskId);
+ this.rowPage = new UnsafeCarbonRowPage(tableFieldStat, baseBlock, !isMemoryAvailable, taskId);
// Delete if any older file exists in sort temp folder
deleteSortLocationIfExists();
@@ -178,7 +183,7 @@ public class UnsafeSortDataRows {
private void addBatch(Object[][] rowBatch, int size) throws CarbonSortKeyAndGroupByException {
for (int i = 0; i < size; i++) {
if (rowPage.canAdd()) {
- bytesAdded += rowPage.addRow(rowBatch[i]);
+ bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -194,15 +199,8 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(
- parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(),
- parameters.getMeasureDataType(),
- memoryBlock,
- saveToDisk, taskId);
- bytesAdded += rowPage.addRow(rowBatch[i]);
+ rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
+ bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -220,7 +218,7 @@ public class UnsafeSortDataRows {
// if record holder list size is equal to sort buffer size then it will
// sort the list and then write current list data to file
if (rowPage.canAdd()) {
- rowPage.addRow(row);
+ rowPage.addRow(row, rowBuffer.get());
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -235,13 +233,8 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(
- parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount(), parameters.getMeasureColCount(),
- parameters.getMeasureDataType(), memoryBlock,
- saveToDisk, taskId);
- rowPage.addRow(row);
+ rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
+ rowPage.addRow(row, rowBuffer.get());
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -269,7 +262,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(rowPage));
} else {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(rowPage));
+ new UnsafeRowComparatorForNormalDims(rowPage));
}
unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
} else {
@@ -295,10 +288,9 @@ public class UnsafeSortDataRows {
// write number of entries to the file
stream.writeInt(actualSize);
for (int i = 0; i < actualSize; i++) {
- rowPage.fillRow(rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(),
- stream);
+ rowPage.writeRow(
+ rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(), stream);
}
-
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
} finally {
@@ -367,7 +359,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(page));
} else {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(page));
+ new UnsafeRowComparatorForNormalDims(page));
}
if (page.isSaveToDisk()) {
// create a new file every time
@@ -380,7 +372,8 @@ public class UnsafeSortDataRows {
writeDataToFile(page, sortTempFile);
LOGGER.info("Time taken to sort row page with size" + page.getBuffer().getActualSize()
+ " and write is: " + (System.currentTimeMillis() - startTime) + ": location:"
- + sortTempFile);
+ + sortTempFile + ", sort temp file size in MB is "
+ + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
page.freeMemory();
// add sort temp filename to and arrayList. When the list size reaches 20 then
// intermediate merging of sort temp files will be triggered
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
index d02be9b..33342dc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -23,63 +23,25 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
-
- /**
- * mapping of dictionary and no dictionary of sort_columns.
- */
- private boolean[] noDictionarySortColumnMaping;
-
private Object baseObject;
+ private TableFieldStat tableFieldStat;
+ private int dictSizeInMemory;
public UnsafeRowComparator(UnsafeCarbonRowPage rowPage) {
- this.noDictionarySortColumnMaping = rowPage.getNoDictionarySortColumnMapping();
this.baseObject = rowPage.getDataBlock().getBaseObject();
+ this.tableFieldStat = rowPage.getTableFieldStat();
+ this.dictSizeInMemory = (tableFieldStat.getDictSortDimCnt()
+ + tableFieldStat.getDictNoSortDimCnt()) * 4;
}
/**
* Below method will be used to compare two mdkey
*/
public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- int diff = 0;
- long rowA = rowL.address;
- long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
- if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowA + sizeA);
- byte[] byteArr1 = new byte[aShort1];
- sizeA += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowA + sizeA, byteArr1,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort1);
- sizeA += aShort1;
-
- short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowB + sizeB);
- byte[] byteArr2 = new byte[aShort2];
- sizeB += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowB + sizeB, byteArr2,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort2);
- sizeB += aShort2;
-
- int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
- if (difference != 0) {
- return difference;
- }
- } else {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
- sizeB += 4;
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
- }
-
- return diff;
+ return compare(rowL, baseObject, rowR, baseObject);
}
/**
@@ -90,35 +52,40 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
int diff = 0;
long rowA = rowL.address;
long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
+ int sizeInDictPartA = 0;
+
+ int sizeInNonDictPartA = 0;
+ int sizeInDictPartB = 0;
+ int sizeInNonDictPartB = 0;
+ for (boolean isNoDictionary : tableFieldStat.getIsSortColNoDictFlags()) {
if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObjectL, rowA + sizeA);
- byte[] byteArr1 = new byte[aShort1];
- sizeA += 2;
+ short lengthA = CarbonUnsafe.getUnsafe().getShort(baseObjectL,
+ rowA + dictSizeInMemory + sizeInNonDictPartA);
+ byte[] byteArr1 = new byte[lengthA];
+ sizeInNonDictPartA += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectL, rowA + sizeA, byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- aShort1);
- sizeA += aShort1;
+ .copyMemory(baseObjectL, rowA + dictSizeInMemory + sizeInNonDictPartA,
+ byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthA);
+ sizeInNonDictPartA += lengthA;
- short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObjectR, rowB + sizeB);
- byte[] byteArr2 = new byte[aShort2];
- sizeB += 2;
+ short lengthB = CarbonUnsafe.getUnsafe().getShort(baseObjectR,
+ rowB + dictSizeInMemory + sizeInNonDictPartB);
+ byte[] byteArr2 = new byte[lengthB];
+ sizeInNonDictPartB += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectR, rowB + sizeB, byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- aShort2);
- sizeB += aShort2;
+ .copyMemory(baseObjectR, rowB + dictSizeInMemory + sizeInNonDictPartB,
+ byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthB);
+ sizeInNonDictPartB += lengthB;
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
if (difference != 0) {
return difference;
}
} else {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeB);
- sizeB += 4;
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeInDictPartA);
+ sizeInDictPartA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeInDictPartB);
+ sizeInDictPartB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
deleted file mode 100644
index 483dcb2..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
-
-public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbonRow> {
-
- private Object baseObject;
-
- private int numberOfSortColumns;
-
- public UnsafeRowComparatorForNormalDIms(UnsafeCarbonRowPage rowPage) {
- this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.numberOfSortColumns = rowPage.getNoDictionarySortColumnMapping().length;
- }
-
- /**
- * Below method will be used to compare two mdkey
- */
- public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- int diff = 0;
- long rowA = rowL.address;
- long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (int i = 0; i < numberOfSortColumns; i++) {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
- sizeB += 4;
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
-
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
new file mode 100644
index 0000000..e9cfb1c
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
+
+public class UnsafeRowComparatorForNormalDims implements Comparator<UnsafeCarbonRow> {
+
+ private Object baseObject;
+
+ private int numberOfSortColumns;
+
+ public UnsafeRowComparatorForNormalDims(UnsafeCarbonRowPage rowPage) {
+ this.baseObject = rowPage.getDataBlock().getBaseObject();
+ this.numberOfSortColumns = rowPage.getTableFieldStat().getIsSortColNoDictFlags().length;
+ }
+
+ /**
+ * Below method will be used to compare two mdkey
+ */
+ public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
+ int diff = 0;
+ long rowA = rowL.address;
+ long rowB = rowR.address;
+ int sizeA = 0;
+ int sizeB = 0;
+ for (int i = 0; i < numberOfSortColumns; i++) {
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
+ sizeA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
+ sizeB += 4;
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+
+ return diff;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
index 686e855..d790c41 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.loading.sort.unsafe.holder;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
/**
@@ -28,7 +29,7 @@ public interface SortTempChunkHolder extends Comparable<SortTempChunkHolder> {
void readRow() throws CarbonSortKeyAndGroupByException;
- Object[] getRow();
+ IntermediateSortTempRow getRow();
int numberOfRows();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index 6b0cfa6..a776db1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -19,9 +19,10 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeInMemoryIntermediateDataMerger;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
@@ -38,21 +39,18 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage[] rowPages;
- private NewRowComparator comparator;
+ private IntermediateSortTempRowComparator comparator;
- private Object[] currentRow;
-
- private int columnSize;
+ private IntermediateSortTempRow currentRow;
public UnsafeFinalMergePageHolder(UnsafeInMemoryIntermediateDataMerger merger,
- boolean[] noDictSortColumnMapping, int columnSize) {
+ boolean[] noDictSortColumnMapping) {
this.actualSize = merger.getEntryCount();
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(noDictSortColumnMapping);
- this.columnSize = columnSize;
+ this.comparator = new IntermediateSortTempRowComparator(noDictSortColumnMapping);
}
public boolean hasNext() {
@@ -63,12 +61,11 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
}
public void readRow() {
- currentRow = new Object[columnSize];
- rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter], currentRow);
+ currentRow = rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter]);
counter++;
}
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return currentRow;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b41f140/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index 6f05088..cbcbbae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -19,8 +19,9 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
public class UnsafeInmemoryHolder implements SortTempChunkHolder {
@@ -33,21 +34,18 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage rowPage;
- private Object[] currentRow;
+ private IntermediateSortTempRow currentRow;
private long address;
- private NewRowComparator comparator;
+ private IntermediateSortTempRowComparator comparator;
- private int columnSize;
-
- public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize,
- int numberOfSortColumns) {
+ public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(rowPage.getNoDictionarySortColumnMapping());
- this.columnSize = columnSize;
+ this.comparator = new IntermediateSortTempRowComparator(
+ rowPage.getTableFieldStat().getIsSortColNoDictFlags());
}
public boolean hasNext() {
@@ -58,13 +56,12 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
}
public void readRow() {
- currentRow = new Object[columnSize];
address = rowPage.getBuffer().get(counter);
- rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset(), currentRow);
+ currentRow = rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset());
counter++;
}
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return currentRow;
}
[24/54] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad]
Optimization in reading/writing for sort temp row
Posted by ra...@apache.org.
[CARBONDATA-2018][DataLoad] Optimization in reading/writing for sort temp row
Pick up the no-sort fields in the row and pack them as bytes array and skip parsing them during merge sort to reduce CPU consumption
This closes #1792
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/21704cf7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/21704cf7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/21704cf7
Branch: refs/heads/master
Commit: 21704cf747712b86bac03d777891c77a5d75dfd0
Parents: 4589ac5
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Feb 8 14:35:14 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../filter/executer/FalseFilterExecutor.java | 10 +-
.../carbondata/core/util/NonDictionaryUtil.java | 67 +--
.../core/util/path/CarbonTablePath.java | 23 +-
.../presto/util/CarbonDataStoreCreator.scala | 1 -
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../CarbonAlterTableDataTypeChangeCommand.scala | 7 +-
.../schema/CarbonAlterTableRenameCommand.scala | 3 +-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 9 +-
.../loading/row/IntermediateSortTempRow.java | 117 +++++
.../loading/sort/SortStepRowHandler.java | 466 +++++++++++++++++++
.../loading/sort/SortStepRowUtil.java | 103 ----
.../sort/unsafe/UnsafeCarbonRowPage.java | 331 ++-----------
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 +--
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 ---
.../UnsafeRowComparatorForNormalDims.java | 59 +++
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 ++----
.../merger/UnsafeIntermediateFileMerger.java | 118 +----
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../merger/CompactionResultSortProcessor.java | 1 -
.../sort/sortdata/IntermediateFileMerger.java | 95 +---
.../IntermediateSortTempRowComparator.java | 73 +++
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94 ----
.../sortdata/RowComparatorForNormalDims.java | 62 ---
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +---
.../sort/sortdata/SortTempFileChunkHolder.java | 174 ++-----
.../sort/sortdata/TableFieldStat.java | 176 +++++++
.../processing/util/CarbonLoaderUtil.java | 4 +-
34 files changed, 1214 insertions(+), 1322 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
index 1c918bc..ee3b0fc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
@@ -44,17 +44,19 @@ public class FalseFilterExecutor implements FilterExecuter {
return group;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return new BitSet();
}
- @Override public void readColumnChunks(RawBlockletColumnChunks blockChunkHolder) {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks blockChunkHolder) {
// Do Nothing
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index d6ecfbc..fca1244 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,18 +82,26 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required Dimension from obj []
+ * Method to get the required dictionary Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static Integer getDimension(int index, Object[] row) {
-
- Integer[] dimensions = (Integer[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
-
+ public static int getDictDimension(int index, Object[] row) {
+ int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
return dimensions[index];
+ }
+ /**
+ * Method to get the required non-dictionary & complex from 3-parted row
+ * @param index
+ * @param row
+ * @return
+ */
+ public static byte[] getNoDictOrComplex(int index, Object[] row) {
+ byte[][] nonDictArray = (byte[][]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+ return nonDictArray[index];
}
/**
@@ -108,60 +116,11 @@ public class NonDictionaryUtil {
return measures[index];
}
- public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
-
- return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
- }
-
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][] byteBufferArr,
Object[] measureArray) {
-
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
-
- /**
- * This method will extract the single dimension from the complete high card dims byte[].+ *
- * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
- *
- * @param highCardArr
- * @param index
- * @param highCardinalityCount
- * @param outBuffer
- */
- public static void extractSingleHighCardDims(byte[] highCardArr, int index,
- int highCardinalityCount, ByteBuffer outBuffer) {
- ByteBuffer buff = null;
- short secIndex = 0;
- short firstIndex = 0;
- int length;
- // if the requested index is a last one then we need to calculate length
- // based on byte[] length.
- if (index == highCardinalityCount - 1) {
- // need to read 2 bytes(1 short) to determine starting offset and
- // length can be calculated by array length.
- buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 2);
- } else {
- // need to read 4 bytes(2 short) to determine starting offset and
- // length.
- buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 4);
- }
-
- firstIndex = buff.getShort();
- // if it is a last dimension in high card then this will be last
- // offset.so calculate length from total length
- if (index == highCardinalityCount - 1) {
- secIndex = (short) highCardArr.length;
- } else {
- secIndex = buff.getShort();
- }
-
- length = secIndex - firstIndex;
-
- outBuffer.position(firstIndex);
- outBuffer.limit(outBuffer.position() + length);
-
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index b7b5e43..50c5a31 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -36,13 +36,13 @@ public class CarbonTablePath {
private static final String DICTIONARY_META_EXT = ".dictmeta";
private static final String SORT_INDEX_EXT = ".sortindex";
private static final String SCHEMA_FILE = "schema";
- public static final String TABLE_STATUS_FILE = "tablestatus";
private static final String FACT_DIR = "Fact";
private static final String SEGMENT_PREFIX = "Segment_";
private static final String PARTITION_PREFIX = "Part";
private static final String DATA_PART_PREFIX = "part-";
private static final String BATCH_PREFIX = "_batchno";
+ public static final String TABLE_STATUS_FILE = "tablestatus";
public static final String CARBON_DATA_EXT = ".carbondata";
public static final String INDEX_FILE_EXT = ".carbonindex";
public static final String MERGE_INDEX_FILE_EXT = ".carbonindexmerge";
@@ -140,20 +140,6 @@ public class CarbonTablePath {
}
/**
- * Return metadata path based on `tablePath`
- */
- public static String getTableStatusPath(String tablePath) {
- return getMetadataPath(tablePath) + File.separator + TABLE_STATUS_FILE;
- }
-
- /**
- * Return table status file path based on `tablePath`
- */
- public static String getTableStatusFilePath(String tablePath) {
- return getMetadataPath(tablePath) + CarbonCommonConstants.FILE_SEPARATOR + TABLE_STATUS_FILE;
- }
-
- /**
* Return absolute path of dictionary meta file
*/
public static String getExternalDictionaryMetaFilePath(String dictionaryPath, String columnId) {
@@ -213,6 +199,13 @@ public class CarbonTablePath {
}
}
+ /**
+ * Return absolute path of table status file
+ */
+ public static String getTableStatusFilePath(String tablePath) {
+ return getMetadataPath(tablePath) + File.separator + TABLE_STATUS_FILE;
+ }
+
public static String getTableStatusFilePathWithUUID(String tablePath, String uuid) {
if (!uuid.isEmpty()) {
return getTableStatusFilePath(tablePath) + CarbonCommonConstants.UNDERSCORE + uuid;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index e768660..1bc9812 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -383,7 +383,6 @@ object CarbonDataStoreCreator {
.getInstance.createCache(CacheType.REVERSE_DICTIONARY)
for (i <- set.indices) {
- // val dim = getDimension(dims, i).get
val columnIdentifier: ColumnIdentifier =
new ColumnIdentifier(dims.get(i).getColumnId, null, null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 5124247..0422239 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.parser.impl.RowParserImpl
-import org.apache.carbondata.processing.loading.sort.SortStepRowUtil
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler
import org.apache.carbondata.processing.loading.steps.DataWriterProcessorStepImpl
import org.apache.carbondata.processing.sort.sortdata.SortParameters
import org.apache.carbondata.processing.store.{CarbonFactHandler, CarbonFactHandlerFactory}
@@ -206,7 +206,7 @@ object DataLoadProcessorStepOnSpark {
val model: CarbonLoadModel = modelBroadcast.value.getCopyWithTaskNo(index.toString)
val conf = DataLoadProcessBuilder.createConfiguration(model)
val sortParameters = SortParameters.createSortParameters(conf)
- val sortStepRowUtil = new SortStepRowUtil(sortParameters)
+ val sortStepRowHandler = new SortStepRowHandler(sortParameters)
TaskContext.get().addTaskFailureListener { (t: TaskContext, e: Throwable) =>
wrapException(e, model)
}
@@ -216,7 +216,7 @@ object DataLoadProcessorStepOnSpark {
override def next(): CarbonRow = {
val row =
- new CarbonRow(sortStepRowUtil.convertRow(rows.next().getData))
+ new CarbonRow(sortStepRowHandler.convertRawRowTo3Parts(rows.next().getData))
rowCounter.add(1)
row
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index 8b0d75f..4ef0df9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -73,10 +73,11 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
throwMetadataException(dbName, tableName, s"Invalid Column: $columnName")
}
// read the latest schema file
- val tableInfo: TableInfo = metastore.getThriftTableInfo(carbonTable)(sparkSession)
+ val tableInfo: org.apache.carbondata.format.TableInfo =
+ metastore.getThriftTableInfo(carbonTable)(sparkSession)
// maintain the added column for schema evolution history
- var addColumnSchema: ColumnSchema = null
- var deletedColumnSchema: ColumnSchema = null
+ var addColumnSchema: org.apache.carbondata.format.ColumnSchema = null
+ var deletedColumnSchema: org.apache.carbondata.format.ColumnSchema = null
val columnSchemaList = tableInfo.fact_table.table_columns.asScala.filter(!_.isInvisible)
columnSchemaList.foreach { columnSchema =>
if (columnSchema.column_name.equalsIgnoreCase(columnName)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 753e637..9fde675 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -168,7 +168,8 @@ private[sql] case class CarbonAlterTableRenameCommand(
updatedParts)
}
- newTablePath = metastore.updateTableSchemaForAlter(newTableIdentifier,
+ newTablePath = metastore.updateTableSchemaForAlter(
+ newTableIdentifier,
carbonTable.getCarbonTableIdentifier,
tableInfo,
schemaEvolutionEntry,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 2c55e12..fc5bdac 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -67,7 +67,8 @@ trait CarbonMetaStore {
* @param carbonStorePath
* @param sparkSession
*/
- def updateTableSchemaForAlter(newTableIdentifier: CarbonTableIdentifier,
+ def updateTableSchemaForAlter(
+ newTableIdentifier: CarbonTableIdentifier,
oldTableIdentifier: CarbonTableIdentifier,
thriftTableInfo: org.apache.carbondata.format.TableInfo,
schemaEvolutionEntry: SchemaEvolutionEntry,
@@ -126,7 +127,8 @@ trait CarbonMetaStore {
def removeTableFromMetadata(dbName: String, tableName: String): Unit
def updateMetadataByThriftTable(schemaFilePath: String,
- tableInfo: TableInfo, dbName: String, tableName: String, tablePath: String): Unit
+ tableInfo: org.apache.carbondata.format.TableInfo,
+ dbName: String, tableName: String, tablePath: String): Unit
def isTablePathExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean
@@ -141,7 +143,8 @@ trait CarbonMetaStore {
def listAllTables(sparkSession: SparkSession): Seq[CarbonTable]
- def getThriftTableInfo(carbonTable: CarbonTable)(sparkSession: SparkSession): TableInfo
+ def getThriftTableInfo(carbonTable: CarbonTable)(sparkSession: SparkSession):
+ org.apache.carbondata.format.TableInfo
def getTableFromMetadataCache(database: String, tableName: String): Option[CarbonTable]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
new file mode 100644
index 0000000..8d351cf
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.loading.row;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+/**
+ * During sort procedure, each row will be written to sort temp file in this logic format.
+ * an intermediate sort temp row consists 3 parts:
+ * dictSort, noDictSort, noSortDimsAndMeasures(dictNoSort, noDictNoSort, measure)
+ */
+public class IntermediateSortTempRow {
+ private int[] dictSortDims;
+ private byte[][] noDictSortDims;
+ private byte[] noSortDimsAndMeasures;
+
+ public IntermediateSortTempRow(int[] dictSortDims, byte[][] noDictSortDims,
+ byte[] noSortDimsAndMeasures) {
+ this.dictSortDims = dictSortDims;
+ this.noDictSortDims = noDictSortDims;
+ this.noSortDimsAndMeasures = noSortDimsAndMeasures;
+ }
+
+ public int[] getDictSortDims() {
+ return dictSortDims;
+ }
+
+ public byte[][] getNoDictSortDims() {
+ return noDictSortDims;
+ }
+
+ public byte[] getNoSortDimsAndMeasures() {
+ return noSortDimsAndMeasures;
+ }
+
+ /**
+ * deserialize from bytes array to get the no sort fields
+ * @param outDictNoSort stores the dict & no-sort fields
+ * @param outNoDictNoSort stores the no-dict & no-sort fields, including complex
+ * @param outMeasures stores the measure fields
+ * @param dataTypes data type for the measure
+ */
+ public void unpackNoSortFromBytes(int[] outDictNoSort, byte[][] outNoDictNoSort,
+ Object[] outMeasures, DataType[] dataTypes) {
+ ByteBuffer rowBuffer = ByteBuffer.wrap(noSortDimsAndMeasures);
+ // read dict_no_sort
+ int dictNoSortCnt = outDictNoSort.length;
+ for (int i = 0; i < dictNoSortCnt; i++) {
+ outDictNoSort[i] = rowBuffer.getInt();
+ }
+
+ // read no_dict_no_sort (including complex)
+ int noDictNoSortCnt = outNoDictNoSort.length;
+ for (int i = 0; i < noDictNoSortCnt; i++) {
+ short len = rowBuffer.getShort();
+ byte[] bytes = new byte[len];
+ rowBuffer.get(bytes);
+ outNoDictNoSort[i] = bytes;
+ }
+
+ // read measure
+ int measureCnt = outMeasures.length;
+ DataType tmpDataType;
+ Object tmpContent;
+ for (short idx = 0 ; idx < measureCnt; idx++) {
+ if ((byte) 0 == rowBuffer.get()) {
+ outMeasures[idx] = null;
+ continue;
+ }
+
+ tmpDataType = dataTypes[idx];
+ if (DataTypes.BOOLEAN == tmpDataType) {
+ if ((byte) 1 == rowBuffer.get()) {
+ tmpContent = true;
+ } else {
+ tmpContent = false;
+ }
+ } else if (DataTypes.SHORT == tmpDataType) {
+ tmpContent = rowBuffer.getShort();
+ } else if (DataTypes.INT == tmpDataType) {
+ tmpContent = rowBuffer.getInt();
+ } else if (DataTypes.LONG == tmpDataType) {
+ tmpContent = rowBuffer.getLong();
+ } else if (DataTypes.DOUBLE == tmpDataType) {
+ tmpContent = rowBuffer.getDouble();
+ } else if (DataTypes.isDecimal(tmpDataType)) {
+ short len = rowBuffer.getShort();
+ byte[] decimalBytes = new byte[len];
+ rowBuffer.get(decimalBytes);
+ tmpContent = DataTypeUtil.byteToBigDecimal(decimalBytes);
+ } else {
+ throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
+ }
+ outMeasures[idx] = tmpContent;
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
new file mode 100644
index 0000000..f31a2b9
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
+
+/**
+ * This class is used to convert/write/read row in sort step in carbondata.
+ * It consists the following function:
+ * 1. convert raw row & intermediate sort temp row to 3-parted row
+ * 2. read/write intermediate sort temp row to sort temp file & unsafe memory
+ * 3. write raw row directly to sort temp file & unsafe memory as intermediate sort temp row
+ */
+public class SortStepRowHandler implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private int dictSortDimCnt = 0;
+ private int dictNoSortDimCnt = 0;
+ private int noDictSortDimCnt = 0;
+ private int noDictNoSortDimCnt = 0;
+ private int measureCnt;
+
+ // indices for dict & sort dimension columns
+ private int[] dictSortDimIdx;
+ // indices for dict & no-sort dimension columns
+ private int[] dictNoSortDimIdx;
+ // indices for no-dict & sort dimension columns
+ private int[] noDictSortDimIdx;
+ // indices for no-dict & no-sort dimension columns, including complex columns
+ private int[] noDictNoSortDimIdx;
+ // indices for measure columns
+ private int[] measureIdx;
+
+ private DataType[] dataTypes;
+
+ /**
+ * constructor
+ * @param tableFieldStat table field stat
+ */
+ public SortStepRowHandler(TableFieldStat tableFieldStat) {
+ this.dictSortDimCnt = tableFieldStat.getDictSortDimCnt();
+ this.dictNoSortDimCnt = tableFieldStat.getDictNoSortDimCnt();
+ this.noDictSortDimCnt = tableFieldStat.getNoDictSortDimCnt();
+ this.noDictNoSortDimCnt = tableFieldStat.getNoDictNoSortDimCnt();
+ this.measureCnt = tableFieldStat.getMeasureCnt();
+ this.dictSortDimIdx = tableFieldStat.getDictSortDimIdx();
+ this.dictNoSortDimIdx = tableFieldStat.getDictNoSortDimIdx();
+ this.noDictSortDimIdx = tableFieldStat.getNoDictSortDimIdx();
+ this.noDictNoSortDimIdx = tableFieldStat.getNoDictNoSortDimIdx();
+ this.measureIdx = tableFieldStat.getMeasureIdx();
+ this.dataTypes = tableFieldStat.getMeasureDataType();
+ }
+
+ /**
+ * constructor
+ * @param sortParameters sort parameters
+ */
+ public SortStepRowHandler(SortParameters sortParameters) {
+ this(new TableFieldStat(sortParameters));
+ }
+
+ /**
+ * Convert carbon row from raw format to 3-parted format.
+ * This method is used in global-sort.
+ *
+ * @param row raw row whose length is the same as field number
+ * @return 3-parted row whose length is 3. (1 for dict dims ,1 for non-dict and complex,
+ * 1 for measures)
+ */
+ public Object[] convertRawRowTo3Parts(Object[] row) {
+ Object[] holder = new Object[3];
+ try {
+ int[] dictDims
+ = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
+ byte[][] nonDictArray = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
+ Object[] measures = new Object[this.measureCnt];
+
+ // convert dict & data
+ int idxAcc = 0;
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ dictDims[idxAcc++] = (int) row[this.dictSortDimIdx[idx]];
+ }
+
+ // convert dict & no-sort
+ for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
+ dictDims[idxAcc++] = (int) row[this.dictNoSortDimIdx[idx]];
+ }
+ // convert no-dict & sort
+ idxAcc = 0;
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ nonDictArray[idxAcc++] = (byte[]) row[this.noDictSortDimIdx[idx]];
+ }
+ // convert no-dict & no-sort
+ for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
+ nonDictArray[idxAcc++] = (byte[]) row[this.noDictNoSortDimIdx[idx]];
+ }
+
+ // convert measure data
+ for (int idx = 0; idx < this.measureCnt; idx++) {
+ measures[idx] = row[this.measureIdx[idx]];
+ }
+
+ NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
+ } catch (Exception e) {
+ throw new RuntimeException("Problem while converting row to 3 parts", e);
+ }
+ return holder;
+ }
+
+ /**
+ * Convert intermediate sort temp row to 3-parted row.
+ * This method is used in the final merge sort to feed rows to the next write step.
+ *
+ * @param sortTempRow intermediate sort temp row
+ * @return 3-parted row
+ */
+ public Object[] convertIntermediateSortTempRowTo3Parted(IntermediateSortTempRow sortTempRow) {
+ int[] dictDims
+ = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
+ byte[][] noDictArray
+ = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
+
+ int[] dictNoSortDims = new int[this.dictNoSortDimCnt];
+ byte[][] noDictNoSortDims = new byte[this.noDictNoSortDimCnt][];
+ Object[] measures = new Object[this.measureCnt];
+
+ sortTempRow.unpackNoSortFromBytes(dictNoSortDims, noDictNoSortDims, measures, this.dataTypes);
+
+ // dict dims
+ System.arraycopy(sortTempRow.getDictSortDims(), 0 , dictDims,
+ 0, this.dictSortDimCnt);
+ System.arraycopy(dictNoSortDims, 0, dictDims,
+ this.dictSortDimCnt, this.dictNoSortDimCnt);;
+
+ // no dict dims, including complex
+ System.arraycopy(sortTempRow.getNoDictSortDims(), 0,
+ noDictArray, 0, this.noDictSortDimCnt);
+ System.arraycopy(noDictNoSortDims, 0, noDictArray,
+ this.noDictSortDimCnt, this.noDictNoSortDimCnt);
+
+ // measures are already here
+
+ Object[] holder = new Object[3];
+ NonDictionaryUtil.prepareOutObj(holder, dictDims, noDictArray, measures);
+ return holder;
+ }
+
+ /**
+ * Read intermediate sort temp row from InputStream.
+ * This method is used during the merge sort phase to read row from sort temp file.
+ *
+ * @param inputStream input stream
+ * @return a row that contains three parts
+ * @throws IOException if error occrus while reading from stream
+ */
+ public IntermediateSortTempRow readIntermediateSortTempRowFromInputStream(
+ DataInputStream inputStream) throws IOException {
+ int[] dictSortDims = new int[this.dictSortDimCnt];
+ byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
+
+ // read dict & sort dim data
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ dictSortDims[idx] = inputStream.readInt();
+ }
+
+ // read no-dict & sort data
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ short len = inputStream.readShort();
+ byte[] bytes = new byte[len];
+ inputStream.readFully(bytes);
+ noDictSortDims[idx] = bytes;
+ }
+
+ // read no-dict dims & measures
+ int len = inputStream.readInt();
+ byte[] noSortDimsAndMeasures = new byte[len];
+ inputStream.readFully(noSortDimsAndMeasures);
+
+ return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write intermediate sort temp row to OutputStream
+ * This method is used during the merge sort phase to write row to sort temp file.
+ *
+ * @param sortTempRow intermediate sort temp row
+ * @param outputStream output stream
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeIntermediateSortTempRowToOutputStream(IntermediateSortTempRow sortTempRow,
+ DataOutputStream outputStream) throws IOException {
+ // write dict & sort dim
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ outputStream.writeInt(sortTempRow.getDictSortDims()[idx]);
+ }
+
+ // write no-dict & sort dim
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = sortTempRow.getNoDictSortDims()[idx];
+ outputStream.writeShort(bytes.length);
+ outputStream.write(bytes);
+ }
+
+ // write packed no-sort dim & measure
+ outputStream.writeInt(sortTempRow.getNoSortDimsAndMeasures().length);
+ outputStream.write(sortTempRow.getNoSortDimsAndMeasures());
+ }
+
+ /**
+ * Write raw row as an intermediate sort temp row to sort temp file.
+ * This method is used in the beginning of the sort phase. Comparing with converting raw row to
+ * intermediate sort temp row and then writing the converted one, Writing raw row directly will
+ * save the intermediate trivial loss.
+ * This method use an array backend buffer to save memory allocation. The buffer will be reused
+ * for all rows (per thread).
+ *
+ * @param row raw row
+ * @param outputStream output stream
+ * @param rowBuffer array backend buffer
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeRawRowAsIntermediateSortTempRowToOutputStream(Object[] row,
+ DataOutputStream outputStream, ByteBuffer rowBuffer) throws IOException {
+ // write dict & sort
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ outputStream.writeInt((int) row[this.dictSortDimIdx[idx]]);
+ }
+
+ // write no-dict & sort
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
+ outputStream.writeShort(bytes.length);
+ outputStream.write(bytes);
+ }
+
+ // pack no-sort
+ rowBuffer.clear();
+ packNoSortFieldsToBytes(row, rowBuffer);
+ rowBuffer.flip();
+ int packSize = rowBuffer.limit();
+
+ // write no-sort
+ outputStream.writeInt(packSize);
+ outputStream.write(rowBuffer.array(), 0, packSize);
+ }
+
+ /**
+ * Read intermediate sort temp row from unsafe memory.
+ * This method is used during merge sort phase for off-heap sort.
+ *
+ * @param baseObject base object of memory block
+ * @param address address of the row
+ * @return intermediate sort temp row
+ */
+ public IntermediateSortTempRow readIntermediateSortTempRowFromUnsafeMemory(Object baseObject,
+ long address) {
+ int size = 0;
+
+ int[] dictSortDims = new int[this.dictSortDimCnt];
+ byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
+
+ // read dict & sort dim
+ for (int idx = 0; idx < dictSortDims.length; idx++) {
+ dictSortDims[idx] = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ }
+
+ // read no-dict & sort dim
+ for (int idx = 0; idx < noDictSortDims.length; idx++) {
+ short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ byte[] bytes = new byte[length];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+ size += length;
+ noDictSortDims[idx] = bytes;
+ }
+
+ // read no-sort dims & measures
+ int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ byte[] noSortDimsAndMeasures = new byte[len];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
+
+ return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write intermediate sort temp row directly from unsafe memory to stream.
+ * This method is used at the late beginning of the sort phase to write in-memory pages
+ * to sort temp file. Comparing with reading intermediate sort temp row from memory and then
+ * writing it, Writing directly from memory to stream will save the intermediate trivial loss.
+ *
+ * @param baseObject base object of the memory block
+ * @param address base address of the row
+ * @param outputStream output stream
+ * @throws IOException if error occurs while writing to stream
+ */
+ public void writeIntermediateSortTempRowFromUnsafeMemoryToStream(Object baseObject,
+ long address, DataOutputStream outputStream) throws IOException {
+ int size = 0;
+
+ // dict & sort
+ for (int idx = 0; idx < dictSortDimCnt; idx++) {
+ outputStream.writeInt(CarbonUnsafe.getUnsafe().getInt(baseObject, address + size));
+ size += 4;
+ }
+
+ // no-dict & sort
+ for (int idx = 0; idx < noDictSortDimCnt; idx++) {
+ short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ byte[] bytes = new byte[length];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+ size += length;
+
+ outputStream.writeShort(length);
+ outputStream.write(bytes);
+ }
+
+ // packed no-sort & measure
+ int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ byte[] noSortDimsAndMeasures = new byte[len];
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
+ noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
+ size += len;
+
+ outputStream.writeInt(len);
+ outputStream.write(noSortDimsAndMeasures);
+ }
+
+ /**
+ * Write raw row as an intermediate sort temp row to memory.
+ * This method is used in the beginning of the off-heap sort phase. Comparing with converting
+ * raw row to intermediate sort temp row and then writing the converted one,
+ * Writing raw row directly will save the intermediate trivial loss.
+ * This method use an array backend buffer to save memory allocation. The buffer will be reused
+ * for all rows (per thread).
+ *
+ * @param row raw row
+ * @param baseObject base object of the memory block
+ * @param address base address for the row
+ * @param rowBuffer array backend buffer
+ * @return number of bytes written to memory
+ */
+ public int writeRawRowAsIntermediateSortTempRowToUnsafeMemory(Object[] row,
+ Object baseObject, long address, ByteBuffer rowBuffer) {
+ int size = 0;
+ // write dict & sort
+ for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
+ CarbonUnsafe.getUnsafe()
+ .putInt(baseObject, address + size, (int) row[this.dictSortDimIdx[idx]]);
+ size += 4;
+ }
+
+ // write no-dict & sort
+ for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) bytes.length);
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
+ bytes.length);
+ size += bytes.length;
+ }
+
+ // convert pack no-sort
+ rowBuffer.clear();
+ packNoSortFieldsToBytes(row, rowBuffer);
+ rowBuffer.flip();
+ int packSize = rowBuffer.limit();
+
+ // write no-sort
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, packSize);
+ size += 4;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(rowBuffer.array(), CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
+ packSize);
+ size += packSize;
+ return size;
+ }
+
+ /**
+ * Pack to no-sort fields to byte array
+ *
+ * @param row raw row
+ * @param rowBuffer byte array backend buffer
+ */
+ private void packNoSortFieldsToBytes(Object[] row, ByteBuffer rowBuffer) {
+ // convert dict & no-sort
+ for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
+ rowBuffer.putInt((int) row[this.dictNoSortDimIdx[idx]]);
+ }
+ // convert no-dict & no-sort
+ for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
+ byte[] bytes = (byte[]) row[this.noDictNoSortDimIdx[idx]];
+ rowBuffer.putShort((short) bytes.length);
+ rowBuffer.put(bytes);
+ }
+
+ // convert measure
+ Object tmpValue;
+ DataType tmpDataType;
+ for (int idx = 0; idx < this.measureCnt; idx++) {
+ tmpValue = row[this.measureIdx[idx]];
+ tmpDataType = this.dataTypes[idx];
+ if (null == tmpValue) {
+ rowBuffer.put((byte) 0);
+ continue;
+ }
+ rowBuffer.put((byte) 1);
+ if (DataTypes.BOOLEAN == tmpDataType) {
+ if ((boolean) tmpValue) {
+ rowBuffer.put((byte) 1);
+ } else {
+ rowBuffer.put((byte) 0);
+ }
+ } else if (DataTypes.SHORT == tmpDataType) {
+ rowBuffer.putShort((Short) tmpValue);
+ } else if (DataTypes.INT == tmpDataType) {
+ rowBuffer.putInt((Integer) tmpValue);
+ } else if (DataTypes.LONG == tmpDataType) {
+ rowBuffer.putLong((Long) tmpValue);
+ } else if (DataTypes.DOUBLE == tmpDataType) {
+ rowBuffer.putDouble((Double) tmpValue);
+ } else if (DataTypes.isDecimal(tmpDataType)) {
+ byte[] decimalBytes = DataTypeUtil.bigDecimalToByte((BigDecimal) tmpValue);
+ rowBuffer.putShort((short) decimalBytes.length);
+ rowBuffer.put(decimalBytes);
+ } else {
+ throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
deleted file mode 100644
index c4e4756..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort;
-
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-
-public class SortStepRowUtil {
- private int measureCount;
- private int dimensionCount;
- private int complexDimensionCount;
- private int noDictionaryCount;
- private int[] dictDimIdx;
- private int[] nonDictIdx;
- private int[] measureIdx;
-
- public SortStepRowUtil(SortParameters parameters) {
- this.measureCount = parameters.getMeasureColCount();
- this.dimensionCount = parameters.getDimColCount();
- this.complexDimensionCount = parameters.getComplexDimColCount();
- this.noDictionaryCount = parameters.getNoDictionaryCount();
- boolean[] isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
-
- int index = 0;
- int nonDicIndex = 0;
- int allCount = 0;
-
- // be careful that the default value is 0
- this.dictDimIdx = new int[dimensionCount - noDictionaryCount];
- this.nonDictIdx = new int[noDictionaryCount + complexDimensionCount];
- this.measureIdx = new int[measureCount];
-
- // indices for dict dim columns
- for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
- if (isNoDictionaryDimensionColumn[i]) {
- nonDictIdx[nonDicIndex++] = i;
- } else {
- dictDimIdx[index++] = allCount;
- }
- allCount++;
- }
-
- // indices for non dict dim/complex columns
- for (int i = 0; i < complexDimensionCount; i++) {
- nonDictIdx[nonDicIndex++] = allCount;
- allCount++;
- }
-
- // indices for measure columns
- for (int i = 0; i < measureCount; i++) {
- measureIdx[i] = allCount;
- allCount++;
- }
- }
-
- public Object[] convertRow(Object[] data) {
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
- Object[] holder = new Object[3];
- try {
-
- int[] dictDims = new int[dimensionCount - noDictionaryCount];
- byte[][] nonDictArray = new byte[noDictionaryCount + complexDimensionCount][];
- Object[] measures = new Object[measureCount];
-
- // write dict dim data
- for (int idx = 0; idx < dictDimIdx.length; idx++) {
- dictDims[idx] = (int) data[dictDimIdx[idx]];
- }
-
- // write non dict dim data
- for (int idx = 0; idx < nonDictIdx.length; idx++) {
- nonDictArray[idx] = (byte[]) data[nonDictIdx[idx]];
- }
-
- // write measure data
- for (int idx = 0; idx < measureIdx.length; idx++) {
- measures[idx] = data[measureIdx[idx]];
- }
- NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
-
- // increment number if record read
- } catch (Exception e) {
- throw new RuntimeException("Problem while converting row ", e);
- }
- //return out row
- return holder;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
index e5583c2..7ea5cb3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
@@ -19,35 +19,20 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.Arrays;
+import java.nio.ByteBuffer;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.memory.IntPointerBuffer;
import org.apache.carbondata.core.memory.MemoryBlock;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.memory.UnsafeSortMemoryManager;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
/**
* It can keep the data of prescribed size data in offheap/onheap memory and returns it when needed
*/
public class UnsafeCarbonRowPage {
-
- private boolean[] noDictionaryDimensionMapping;
-
- private boolean[] noDictionarySortColumnMapping;
-
- private int dimensionSize;
-
- private int measureSize;
-
- private DataType[] measureDataType;
-
- private long[] nullSetWords;
-
private IntPointerBuffer buffer;
private int lastSize;
@@ -62,16 +47,14 @@ public class UnsafeCarbonRowPage {
private long taskId;
- public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
- boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, DataType[] type,
- MemoryBlock memoryBlock, boolean saveToDisk, long taskId) {
- this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
- this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
- this.dimensionSize = dimensionSize;
- this.measureSize = measureSize;
- this.measureDataType = type;
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+
+ public UnsafeCarbonRowPage(TableFieldStat tableFieldStat, MemoryBlock memoryBlock,
+ boolean saveToDisk, long taskId) {
+ this.tableFieldStat = tableFieldStat;
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.saveToDisk = saveToDisk;
- this.nullSetWords = new long[((measureSize - 1) >> 6) + 1];
this.taskId = taskId;
buffer = new IntPointerBuffer(this.taskId);
this.dataBlock = memoryBlock;
@@ -80,255 +63,44 @@ public class UnsafeCarbonRowPage {
this.managerType = MemoryManagerType.UNSAFE_MEMORY_MANAGER;
}
- public int addRow(Object[] row) {
- int size = addRow(row, dataBlock.getBaseOffset() + lastSize);
+ public int addRow(Object[] row, ByteBuffer rowBuffer) {
+ int size = addRow(row, dataBlock.getBaseOffset() + lastSize, rowBuffer);
buffer.set(lastSize);
lastSize = lastSize + size;
return size;
}
- private int addRow(Object[] row, long address) {
- if (row == null) {
- throw new RuntimeException("Row is null ??");
- }
- int dimCount = 0;
- int size = 0;
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- } else {
- int value = (int) row[dimCount];
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, value);
- size += 4;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- }
- Arrays.fill(nullSetWords, 0);
- int nullSetSize = nullSetWords.length * 8;
- int nullWordLoc = size;
- size += nullSetSize;
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- Object value = row[mesCount + dimensionSize];
- if (null != value) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- Boolean bval = (Boolean) value;
- CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, bval);
- size += 1;
- } else if (dataType == DataTypes.SHORT) {
- Short sval = (Short) value;
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
- size += 2;
- } else if (dataType == DataTypes.INT) {
- Integer ival = (Integer) value;
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival);
- size += 4;
- } else if (dataType == DataTypes.LONG) {
- Long val = (Long) value;
- CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val);
- size += 8;
- } else if (dataType == DataTypes.DOUBLE) {
- Double doubleVal = (Double) value;
- CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal);
- size += 8;
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal decimalVal = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) bigDecimalInBytes.length);
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- set(nullSetWords, mesCount);
- } else {
- unset(nullSetWords, mesCount);
- }
- }
- CarbonUnsafe.getUnsafe().copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
- address + nullWordLoc, nullSetSize);
- return size;
+ /**
+ * add raw row as intermidiate sort temp row to page
+ *
+ * @param row
+ * @param address
+ * @return
+ */
+ private int addRow(Object[] row, long address, ByteBuffer rowBuffer) {
+ return sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToUnsafeMemory(row,
+ dataBlock.getBaseObject(), address, rowBuffer);
}
- public Object[] getRow(long address, Object[] rowToFill) {
- int dimCount = 0;
- int size = 0;
-
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- col.length);
- size += col.length;
- rowToFill[dimCount] = col;
- } else {
- int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- rowToFill[dimCount] = anInt;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
- size += col.length;
- rowToFill[dimCount] = col;
- }
-
- int nullSetSize = nullSetWords.length * 8;
- Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
- nullSetSize);
- size += nullSetSize;
-
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- if (isSet(nullSetWords, mesCount)) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- Boolean bval = CarbonUnsafe.getUnsafe().getBoolean(baseObject, address + size);
- size += 1;
- rowToFill[dimensionSize + mesCount] = bval;
- } else if (dataType == DataTypes.SHORT) {
- Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- rowToFill[dimensionSize + mesCount] = sval;
- } else if (dataType == DataTypes.INT) {
- Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- rowToFill[dimensionSize + mesCount] = ival;
- } else if (dataType == DataTypes.LONG) {
- Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = val;
- } else if (dataType == DataTypes.DOUBLE) {
- Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
- size += 8;
- rowToFill[dimensionSize + mesCount] = doubleVal;
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- } else {
- rowToFill[dimensionSize + mesCount] = null;
- }
- }
- return rowToFill;
+ /**
+ * get one row from memory address
+ * @param address address
+ * @return one row
+ */
+ public IntermediateSortTempRow getRow(long address) {
+ return sortStepRowHandler.readIntermediateSortTempRowFromUnsafeMemory(
+ dataBlock.getBaseObject(), address);
}
- public void fillRow(long address, DataOutputStream stream) throws IOException {
- int dimCount = 0;
- int size = 0;
-
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- col.length);
- size += col.length;
- stream.writeShort(aShort);
- stream.write(col);
- } else {
- int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- stream.writeInt(anInt);
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] col = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
- size += col.length;
- stream.writeShort(aShort);
- stream.write(col);
- }
-
- int nullSetSize = nullSetWords.length * 8;
- Arrays.fill(nullSetWords, 0);
- CarbonUnsafe.getUnsafe()
- .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
- nullSetSize);
- size += nullSetSize;
- for (int i = 0; i < nullSetWords.length; i++) {
- stream.writeLong(nullSetWords[i]);
- }
-
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- if (isSet(nullSetWords, mesCount)) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.SHORT) {
- short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- stream.writeShort(sval);
- } else if (dataType == DataTypes.INT) {
- int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- stream.writeInt(ival);
- } else if (dataType == DataTypes.LONG) {
- long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
- size += 8;
- stream.writeLong(val);
- } else if (dataType == DataTypes.DOUBLE) {
- double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
- size += 8;
- stream.writeDouble(doubleVal);
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- byte[] bigDecimalInBytes = new byte[aShort];
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- stream.writeShort(aShort);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- }
- }
+ /**
+ * write a row to stream
+ * @param address address of a row
+ * @param stream stream
+ * @throws IOException
+ */
+ public void writeRow(long address, DataOutputStream stream) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowFromUnsafeMemoryToStream(
+ dataBlock.getBaseObject(), address, stream);
}
public void freeMemory() {
@@ -362,27 +134,8 @@ public class UnsafeCarbonRowPage {
return dataBlock;
}
- public static void set(long[] words, int index) {
- int wordOffset = (index >> 6);
- words[wordOffset] |= (1L << index);
- }
-
- public static void unset(long[] words, int index) {
- int wordOffset = (index >> 6);
- words[wordOffset] &= ~(1L << index);
- }
-
- public static boolean isSet(long[] words, int index) {
- int wordOffset = (index >> 6);
- return ((words[wordOffset] & (1L << index)) != 0);
- }
-
- public boolean[] getNoDictionaryDimensionMapping() {
- return noDictionaryDimensionMapping;
- }
-
- public boolean[] getNoDictionarySortColumnMapping() {
- return noDictionarySortColumnMapping;
+ public TableFieldStat getTableFieldStat() {
+ return tableFieldStat;
}
public void setNewDataBlock(MemoryBlock newMemoryBlock) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index 4dd5e44..5d038d3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -41,13 +42,14 @@ import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparator;
-import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDIms;
+import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDims;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.UnsafeIntSortDataFormat;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class UnsafeSortDataRows {
@@ -69,7 +71,8 @@ public class UnsafeSortDataRows {
*/
private SortParameters parameters;
-
+ private TableFieldStat tableFieldStat;
+ private ThreadLocal<ByteBuffer> rowBuffer;
private UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger;
private UnsafeCarbonRowPage rowPage;
@@ -94,7 +97,13 @@ public class UnsafeSortDataRows {
public UnsafeSortDataRows(SortParameters parameters,
UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger, int inMemoryChunkSize) {
this.parameters = parameters;
-
+ this.tableFieldStat = new TableFieldStat(parameters);
+ this.rowBuffer = new ThreadLocal<ByteBuffer>() {
+ @Override protected ByteBuffer initialValue() {
+ byte[] backedArray = new byte[2 * 1024 * 1024];
+ return ByteBuffer.wrap(backedArray);
+ }
+ };
this.unsafeInMemoryIntermediateFileMerger = unsafeInMemoryIntermediateFileMerger;
// observer of writing file in thread
@@ -127,11 +136,7 @@ public class UnsafeSortDataRows {
if (isMemoryAvailable) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size());
}
- this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(), parameters.getMeasureDataType(), baseBlock,
- !isMemoryAvailable, taskId);
+ this.rowPage = new UnsafeCarbonRowPage(tableFieldStat, baseBlock, !isMemoryAvailable, taskId);
// Delete if any older file exists in sort temp folder
deleteSortLocationIfExists();
@@ -178,7 +183,7 @@ public class UnsafeSortDataRows {
private void addBatch(Object[][] rowBatch, int size) throws CarbonSortKeyAndGroupByException {
for (int i = 0; i < size; i++) {
if (rowPage.canAdd()) {
- bytesAdded += rowPage.addRow(rowBatch[i]);
+ bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -194,15 +199,8 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(
- parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount(),
- parameters.getMeasureColCount(),
- parameters.getMeasureDataType(),
- memoryBlock,
- saveToDisk, taskId);
- bytesAdded += rowPage.addRow(rowBatch[i]);
+ rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
+ bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -220,7 +218,7 @@ public class UnsafeSortDataRows {
// if record holder list size is equal to sort buffer size then it will
// sort the list and then write current list data to file
if (rowPage.canAdd()) {
- rowPage.addRow(row);
+ rowPage.addRow(row, rowBuffer.get());
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -235,13 +233,8 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(
- parameters.getNoDictionaryDimnesionColumn(),
- parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount(), parameters.getMeasureColCount(),
- parameters.getMeasureDataType(), memoryBlock,
- saveToDisk, taskId);
- rowPage.addRow(row);
+ rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
+ rowPage.addRow(row, rowBuffer.get());
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -269,7 +262,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(rowPage));
} else {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(rowPage));
+ new UnsafeRowComparatorForNormalDims(rowPage));
}
unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
} else {
@@ -295,10 +288,9 @@ public class UnsafeSortDataRows {
// write number of entries to the file
stream.writeInt(actualSize);
for (int i = 0; i < actualSize; i++) {
- rowPage.fillRow(rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(),
- stream);
+ rowPage.writeRow(
+ rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(), stream);
}
-
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
} finally {
@@ -367,7 +359,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(page));
} else {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(page));
+ new UnsafeRowComparatorForNormalDims(page));
}
if (page.isSaveToDisk()) {
// create a new file every time
@@ -380,7 +372,8 @@ public class UnsafeSortDataRows {
writeDataToFile(page, sortTempFile);
LOGGER.info("Time taken to sort row page with size" + page.getBuffer().getActualSize()
+ " and write is: " + (System.currentTimeMillis() - startTime) + ": location:"
- + sortTempFile);
+ + sortTempFile + ", sort temp file size in MB is "
+ + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
page.freeMemory();
// add sort temp filename to and arrayList. When the list size reaches 20 then
// intermediate merging of sort temp files will be triggered
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
index d02be9b..33342dc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -23,63 +23,25 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
-
- /**
- * mapping of dictionary and no dictionary of sort_columns.
- */
- private boolean[] noDictionarySortColumnMaping;
-
private Object baseObject;
+ private TableFieldStat tableFieldStat;
+ private int dictSizeInMemory;
public UnsafeRowComparator(UnsafeCarbonRowPage rowPage) {
- this.noDictionarySortColumnMaping = rowPage.getNoDictionarySortColumnMapping();
this.baseObject = rowPage.getDataBlock().getBaseObject();
+ this.tableFieldStat = rowPage.getTableFieldStat();
+ this.dictSizeInMemory = (tableFieldStat.getDictSortDimCnt()
+ + tableFieldStat.getDictNoSortDimCnt()) * 4;
}
/**
* Below method will be used to compare two mdkey
*/
public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- int diff = 0;
- long rowA = rowL.address;
- long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
- if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowA + sizeA);
- byte[] byteArr1 = new byte[aShort1];
- sizeA += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowA + sizeA, byteArr1,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort1);
- sizeA += aShort1;
-
- short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowB + sizeB);
- byte[] byteArr2 = new byte[aShort2];
- sizeB += 2;
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowB + sizeB, byteArr2,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort2);
- sizeB += aShort2;
-
- int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
- if (difference != 0) {
- return difference;
- }
- } else {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
- sizeB += 4;
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
- }
-
- return diff;
+ return compare(rowL, baseObject, rowR, baseObject);
}
/**
@@ -90,35 +52,40 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
int diff = 0;
long rowA = rowL.address;
long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
+ int sizeInDictPartA = 0;
+
+ int sizeInNonDictPartA = 0;
+ int sizeInDictPartB = 0;
+ int sizeInNonDictPartB = 0;
+ for (boolean isNoDictionary : tableFieldStat.getIsSortColNoDictFlags()) {
if (isNoDictionary) {
- short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObjectL, rowA + sizeA);
- byte[] byteArr1 = new byte[aShort1];
- sizeA += 2;
+ short lengthA = CarbonUnsafe.getUnsafe().getShort(baseObjectL,
+ rowA + dictSizeInMemory + sizeInNonDictPartA);
+ byte[] byteArr1 = new byte[lengthA];
+ sizeInNonDictPartA += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectL, rowA + sizeA, byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- aShort1);
- sizeA += aShort1;
+ .copyMemory(baseObjectL, rowA + dictSizeInMemory + sizeInNonDictPartA,
+ byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthA);
+ sizeInNonDictPartA += lengthA;
- short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObjectR, rowB + sizeB);
- byte[] byteArr2 = new byte[aShort2];
- sizeB += 2;
+ short lengthB = CarbonUnsafe.getUnsafe().getShort(baseObjectR,
+ rowB + dictSizeInMemory + sizeInNonDictPartB);
+ byte[] byteArr2 = new byte[lengthB];
+ sizeInNonDictPartB += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectR, rowB + sizeB, byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET,
- aShort2);
- sizeB += aShort2;
+ .copyMemory(baseObjectR, rowB + dictSizeInMemory + sizeInNonDictPartB,
+ byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthB);
+ sizeInNonDictPartB += lengthB;
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
if (difference != 0) {
return difference;
}
} else {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeB);
- sizeB += 4;
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeInDictPartA);
+ sizeInDictPartA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeInDictPartB);
+ sizeInDictPartB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
deleted file mode 100644
index 483dcb2..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
-
-public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbonRow> {
-
- private Object baseObject;
-
- private int numberOfSortColumns;
-
- public UnsafeRowComparatorForNormalDIms(UnsafeCarbonRowPage rowPage) {
- this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.numberOfSortColumns = rowPage.getNoDictionarySortColumnMapping().length;
- }
-
- /**
- * Below method will be used to compare two mdkey
- */
- public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- int diff = 0;
- long rowA = rowL.address;
- long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (int i = 0; i < numberOfSortColumns; i++) {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
- sizeB += 4;
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
-
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
new file mode 100644
index 0000000..e9cfb1c
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
+
+public class UnsafeRowComparatorForNormalDims implements Comparator<UnsafeCarbonRow> {
+
+ private Object baseObject;
+
+ private int numberOfSortColumns;
+
+ public UnsafeRowComparatorForNormalDims(UnsafeCarbonRowPage rowPage) {
+ this.baseObject = rowPage.getDataBlock().getBaseObject();
+ this.numberOfSortColumns = rowPage.getTableFieldStat().getIsSortColNoDictFlags().length;
+ }
+
+ /**
+ * Below method will be used to compare two mdkey
+ */
+ public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
+ int diff = 0;
+ long rowA = rowL.address;
+ long rowB = rowR.address;
+ int sizeA = 0;
+ int sizeB = 0;
+ for (int i = 0; i < numberOfSortColumns; i++) {
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
+ sizeA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
+ sizeB += 4;
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+
+ return diff;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
index 686e855..d790c41 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.processing.loading.sort.unsafe.holder;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
/**
@@ -28,7 +29,7 @@ public interface SortTempChunkHolder extends Comparable<SortTempChunkHolder> {
void readRow() throws CarbonSortKeyAndGroupByException;
- Object[] getRow();
+ IntermediateSortTempRow getRow();
int numberOfRows();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index 6b0cfa6..a776db1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -19,9 +19,10 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeInMemoryIntermediateDataMerger;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
@@ -38,21 +39,18 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage[] rowPages;
- private NewRowComparator comparator;
+ private IntermediateSortTempRowComparator comparator;
- private Object[] currentRow;
-
- private int columnSize;
+ private IntermediateSortTempRow currentRow;
public UnsafeFinalMergePageHolder(UnsafeInMemoryIntermediateDataMerger merger,
- boolean[] noDictSortColumnMapping, int columnSize) {
+ boolean[] noDictSortColumnMapping) {
this.actualSize = merger.getEntryCount();
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(noDictSortColumnMapping);
- this.columnSize = columnSize;
+ this.comparator = new IntermediateSortTempRowComparator(noDictSortColumnMapping);
}
public boolean hasNext() {
@@ -63,12 +61,11 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
}
public void readRow() {
- currentRow = new Object[columnSize];
- rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter], currentRow);
+ currentRow = rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter]);
counter++;
}
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return currentRow;
}
[53/54] [abbrv] carbondata git commit: [HOTFIX] Add dava doc for
datamap interface
Posted by ra...@apache.org.
[HOTFIX] Add dava doc for datamap interface
1. Rename some of the datamap interface
2. Add more java doc for all public class of datamap interface
This closes #1998
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/fc2a7eb3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/fc2a7eb3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/fc2a7eb3
Branch: refs/heads/master
Commit: fc2a7eb36c06aaced9f0dddcaefefddad1f2b331
Parents: 89a12af
Author: Jacky Li <ja...@qq.com>
Authored: Mon Feb 26 10:04:51 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapChooser.java | 8 +-
.../core/datamap/DataMapDistributable.java | 2 +
.../carbondata/core/datamap/DataMapLevel.java | 37 +
.../carbondata/core/datamap/DataMapMeta.java | 7 +
.../core/datamap/DataMapRegistry.java | 21 +
.../core/datamap/DataMapStoreManager.java | 86 +-
.../carbondata/core/datamap/DataMapType.java | 21 -
.../carbondata/core/datamap/TableDataMap.java | 54 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 111 ---
.../core/datamap/dev/BlockletSerializer.java | 8 +-
.../carbondata/core/datamap/dev/DataMap.java | 58 ++
.../core/datamap/dev/DataMapFactory.java | 86 ++
.../core/datamap/dev/DataMapWriter.java | 117 +++
.../core/datamap/dev/IndexDataMap.java | 56 --
.../core/datamap/dev/IndexDataMapFactory.java | 86 --
.../AbstractCoarseGrainIndexDataMap.java | 24 -
.../AbstractCoarseGrainIndexDataMapFactory.java | 34 -
.../dev/cgdatamap/CoarseGrainDataMap.java | 31 +
.../cgdatamap/CoarseGrainDataMapFactory.java | 38 +
.../datamap/dev/expr/AndDataMapExprWrapper.java | 4 +-
.../datamap/dev/expr/DataMapExprWrapper.java | 4 +-
.../dev/expr/DataMapExprWrapperImpl.java | 6 +-
.../datamap/dev/expr/OrDataMapExprWrapper.java | 4 +-
.../AbstractFineGrainIndexDataMap.java | 24 -
.../AbstractFineGrainIndexDataMapFactory.java | 38 -
.../dev/fgdatamap/FineGrainBlocklet.java | 134 +++
.../datamap/dev/fgdatamap/FineGrainDataMap.java | 30 +
.../dev/fgdatamap/FineGrainDataMapFactory.java | 44 +
.../indexstore/BlockletDataMapIndexStore.java | 30 +-
.../core/indexstore/FineGrainBlocklet.java | 128 ---
.../blockletindex/BlockletDataMap.java | 971 +++++++++++++++++++
.../blockletindex/BlockletDataMapFactory.java | 285 ++++++
.../blockletindex/BlockletDataMapModel.java | 2 +-
.../blockletindex/BlockletDataRefNode.java | 2 +-
.../blockletindex/BlockletIndexDataMap.java | 971 -------------------
.../BlockletIndexDataMapFactory.java | 285 ------
.../core/metadata/schema/table/CarbonTable.java | 2 +-
.../blockletindex/TestBlockletDataMap.java | 59 ++
.../blockletindex/TestBlockletIndexDataMap.java | 59 --
.../datamap/examples/MinMaxDataWriter.java | 4 +-
.../datamap/examples/MinMaxIndexDataMap.java | 4 +-
.../examples/MinMaxIndexDataMapFactory.java | 15 +-
.../hadoop/api/CarbonTableInputFormat.java | 8 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 381 ++++++++
.../datamap/CGIndexDataMapTestCase.scala | 383 --------
.../testsuite/datamap/DataMapWriterSuite.scala | 216 +++++
.../testsuite/datamap/FGDataMapTestCase.scala | 473 +++++++++
.../datamap/FGIndexDataMapTestCase.scala | 472 ---------
.../datamap/IndexDataMapWriterSuite.scala | 217 -----
.../testsuite/datamap/TestDataMapCommand.scala | 285 ++++++
.../datamap/TestIndexDataMapCommand.scala | 285 ------
.../testsuite/sortcolumns/TestSortColumns.scala | 5 +
.../sortcolumns/TestSortColumnsWithUnsafe.scala | 15 +-
.../carbondata/datamap/DataMapProvider.java | 12 +-
.../datamap/IndexDataMapProvider.java | 20 +-
.../datamap/DataMapWriterListener.java | 55 +-
56 files changed, 3463 insertions(+), 3354 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index 41e9b56..94b48c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -74,12 +74,12 @@ public class DataMapChooser {
Expression expression = resolverIntf.getFilterExpression();
// First check for FG datamaps if any exist
List<TableDataMap> allDataMapFG =
- DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapType.FG);
+ DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.FG);
ExpressionTuple tuple = selectDataMap(expression, allDataMapFG);
if (tuple.dataMapExprWrapper == null) {
// Check for CG datamap
List<TableDataMap> allDataMapCG =
- DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapType.CG);
+ DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.CG);
tuple = selectDataMap(expression, allDataMapCG);
}
if (tuple.dataMapExprWrapper != null) {
@@ -212,10 +212,10 @@ public class DataMapChooser {
List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) {
List<DataMapTuple> tuples = new ArrayList<>();
for (TableDataMap dataMap : allDataMap) {
- if (contains(dataMap.getIndexDataMapFactory().getMeta(), columnExpressions, expressionTypes))
+ if (contains(dataMap.getDataMapFactory().getMeta(), columnExpressions, expressionTypes))
{
tuples.add(
- new DataMapTuple(dataMap.getIndexDataMapFactory().getMeta().getIndexedColumns().size(),
+ new DataMapTuple(dataMap.getDataMapFactory().getMeta().getIndexedColumns().size(),
dataMap));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
index 473d8d4..5cd8c40 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapDistributable.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap;
import java.io.IOException;
import java.io.Serializable;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
@@ -27,6 +28,7 @@ import org.apache.hadoop.mapreduce.InputSplit;
/**
* Distributable class for datamap.
*/
+@InterfaceAudience.Internal
public abstract class DataMapDistributable extends InputSplit
implements Distributable, Serializable {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapLevel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapLevel.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapLevel.java
new file mode 100644
index 0000000..2c9672a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapLevel.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * Index level of the datamap
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public enum DataMapLevel {
+ /**
+ * Coarse Grain Index, index is of blocklet level
+ */
+ CG,
+
+ /**
+ * Fine Grain Index, index is of row level
+ */
+ FG;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
index dd15ccb..396c5db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapMeta.java
@@ -19,8 +19,15 @@ package org.apache.carbondata.core.datamap;
import java.util.List;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+/**
+ * Metadata of the datamap, set by DataMap developer
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
public class DataMapMeta {
private List<String> indexedColumns;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
index 03c0c3e..1b6782a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
@@ -21,6 +21,27 @@ import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * Developer can register a datamap implementation with a short name.
+ * After registration, user can use short name to create the datamap, like
+ * <p>
+ * {@code
+ * CREATE DATAMAP dm ON TABLE table
+ * USING 'short-name-of-the-datamap'
+ * }
+ * otherwise, user should use the class name of the datamap implementation to create the datamap
+ * (subclass of {@link org.apache.carbondata.core.datamap.dev.DataMapFactory})
+ * <p>
+ * {@code
+ * CREATE DATAMAP dm ON TABLE table
+ * USING 'class-name-of-the-datamap'
+ * }
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
public class DataMapRegistry {
private static Map<String, String> shortNameToClassName = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index ed4e18c..e57a841 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -22,14 +22,15 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.MetadataProcessException;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMapFactory;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -41,6 +42,7 @@ import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
/**
* It maintains all the DataMaps in it.
*/
+@InterfaceAudience.Internal
public final class DataMapStoreManager {
private static DataMapStoreManager instance = new DataMapStoreManager();
@@ -63,12 +65,12 @@ public final class DataMapStoreManager {
* It gives all datamaps of type @mapType except the default datamap.
*
*/
- public List<TableDataMap> getAllDataMap(CarbonTable carbonTable, DataMapType mapType) {
+ public List<TableDataMap> getAllDataMap(CarbonTable carbonTable, DataMapLevel mapType) {
List<TableDataMap> dataMaps = new ArrayList<>();
- List<TableDataMap> tableDataMaps = getAllDataMap(carbonTable);
- if (tableDataMaps != null) {
- for (TableDataMap dataMap : tableDataMaps) {
- if (mapType == dataMap.getIndexDataMapFactory().getDataMapType()) {
+ List<TableDataMap> tableIndices = getAllDataMap(carbonTable);
+ if (tableIndices != null) {
+ for (TableDataMap dataMap : tableIndices) {
+ if (mapType == dataMap.getDataMapFactory().getDataMapType()) {
dataMaps.add(dataMap);
}
}
@@ -96,13 +98,13 @@ public final class DataMapStoreManager {
}
/**
- * It gives the default datamap of the table. Default datamap of any table is BlockletIndexDataMap
+ * It gives the default datamap of the table. Default datamap of any table is BlockletDataMap
*
* @param identifier
* @return
*/
public TableDataMap getDefaultDataMap(AbsoluteTableIdentifier identifier) {
- return getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA);
+ return getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
}
/**
@@ -110,16 +112,16 @@ public final class DataMapStoreManager {
*/
public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
- List<TableDataMap> tableDataMaps = allDataMaps.get(table);
+ List<TableDataMap> tableIndices = allDataMaps.get(table);
TableDataMap dataMap = null;
- if (tableDataMaps != null) {
- dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableDataMaps);
+ if (tableIndices != null) {
+ dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableIndices);
}
if (dataMap == null) {
synchronized (table.intern()) {
- tableDataMaps = allDataMaps.get(table);
- if (tableDataMaps != null) {
- dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableDataMaps);
+ tableIndices = allDataMaps.get(table);
+ if (tableIndices != null) {
+ dataMap = getTableDataMap(dataMapSchema.getDataMapName(), tableIndices);
}
if (dataMap == null) {
try {
@@ -143,12 +145,12 @@ public final class DataMapStoreManager {
*/
private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
DataMapSchema dataMapSchema) throws MalformedDataMapCommandException {
- IndexDataMapFactory indexDataMapFactory;
+ DataMapFactory dataMapFactory;
try {
- // try to create datamap by reflection to test whether it is a valid IndexDataMapFactory class
- Class<? extends IndexDataMapFactory> factoryClass =
- (Class<? extends IndexDataMapFactory>) Class.forName(dataMapSchema.getClassName());
- indexDataMapFactory = factoryClass.newInstance();
+ // try to create datamap by reflection to test whether it is a valid DataMapFactory class
+ Class<? extends DataMapFactory> factoryClass =
+ (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ dataMapFactory = factoryClass.newInstance();
} catch (ClassNotFoundException e) {
throw new MalformedDataMapCommandException(
"DataMap '" + dataMapSchema.getClassName() + "' not found");
@@ -156,39 +158,39 @@ public final class DataMapStoreManager {
throw new MetadataProcessException(
"failed to create DataMap '" + dataMapSchema.getClassName() + "'", e);
}
- return registerDataMap(identifier, dataMapSchema, indexDataMapFactory);
+ return registerDataMap(identifier, dataMapSchema, dataMapFactory);
}
public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier,
- DataMapSchema dataMapSchema, IndexDataMapFactory indexDataMapFactory) {
+ DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
- List<TableDataMap> tableDataMaps = allDataMaps.get(table);
- if (tableDataMaps == null) {
- tableDataMaps = new ArrayList<>();
+ List<TableDataMap> tableIndices = allDataMaps.get(table);
+ if (tableIndices == null) {
+ tableIndices = new ArrayList<>();
}
- indexDataMapFactory.init(identifier, dataMapSchema);
+ dataMapFactory.init(identifier, dataMapSchema);
BlockletDetailsFetcher blockletDetailsFetcher;
SegmentPropertiesFetcher segmentPropertiesFetcher = null;
- if (indexDataMapFactory instanceof BlockletDetailsFetcher) {
- blockletDetailsFetcher = (BlockletDetailsFetcher) indexDataMapFactory;
+ if (dataMapFactory instanceof BlockletDetailsFetcher) {
+ blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
} else {
blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
- TableDataMap dataMap = new TableDataMap(identifier, dataMapSchema, indexDataMapFactory,
+ TableDataMap dataMap = new TableDataMap(identifier, dataMapSchema, dataMapFactory,
blockletDetailsFetcher, segmentPropertiesFetcher);
- tableDataMaps.add(dataMap);
- allDataMaps.put(table, tableDataMaps);
+ tableIndices.add(dataMap);
+ allDataMaps.put(table, tableIndices);
return dataMap;
}
- private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableDataMaps) {
+ private TableDataMap getTableDataMap(String dataMapName, List<TableDataMap> tableIndices) {
TableDataMap dataMap = null;
- for (TableDataMap tableDataMap : tableDataMaps) {
+ for (TableDataMap tableDataMap : tableIndices) {
if (tableDataMap.getDataMapSchema().getDataMapName().equals(dataMapName)) {
dataMap = tableDataMap;
break;
@@ -218,10 +220,10 @@ public final class DataMapStoreManager {
*/
public void clearDataMaps(AbsoluteTableIdentifier identifier) {
String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
- List<TableDataMap> tableDataMaps = allDataMaps.get(tableUniqueName);
+ List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName);
segmentRefreshMap.remove(identifier.uniqueName());
- if (tableDataMaps != null) {
- for (TableDataMap tableDataMap : tableDataMaps) {
+ if (tableIndices != null) {
+ for (TableDataMap tableDataMap : tableIndices) {
if (tableDataMap != null) {
tableDataMap.clear();
break;
@@ -237,15 +239,15 @@ public final class DataMapStoreManager {
* @param identifier Table identifier
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps =
+ List<TableDataMap> tableIndices =
allDataMaps.get(identifier.getCarbonTableIdentifier().getTableUniqueName());
- if (tableDataMaps != null) {
+ if (tableIndices != null) {
int i = 0;
- for (TableDataMap tableDataMap : tableDataMaps) {
+ for (TableDataMap tableDataMap : tableIndices) {
if (tableDataMap != null && dataMapName
.equalsIgnoreCase(tableDataMap.getDataMapSchema().getDataMapName())) {
tableDataMap.clear();
- tableDataMaps.remove(i);
+ tableIndices.remove(i);
break;
}
i++;
@@ -260,8 +262,8 @@ public final class DataMapStoreManager {
* @return
*/
private BlockletDetailsFetcher getBlockletDetailsFetcher(AbsoluteTableIdentifier identifier) {
- TableDataMap blockletMap = getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA);
- return (BlockletDetailsFetcher) blockletMap.getIndexDataMapFactory();
+ TableDataMap blockletMap = getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
+ return (BlockletDetailsFetcher) blockletMap.getDataMapFactory();
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
deleted file mode 100644
index bf812b3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapType.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap;
-
-public enum DataMapType {
- CG,FG;
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index b9a50d2..81571ce 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -20,16 +20,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
-import org.apache.carbondata.core.datamap.dev.IndexDataMap;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -40,16 +41,21 @@ import org.apache.carbondata.events.OperationContext;
import org.apache.carbondata.events.OperationEventListener;
/**
- * IndexDataMap at the table level, user can add any number of datamaps for one table. Depends
- * on the filter condition it can prune the blocklets.
+ * Index at the table level, user can add any number of DataMap for one table, by
+ * {@code
+ * CREATE DATAMAP dm ON TABLE table
+ * USING 'class name of DataMapFactory implementation'
+ * }
+ * Depends on the filter condition it can prune the data (blocklet or row level).
*/
+@InterfaceAudience.Internal
public final class TableDataMap extends OperationEventListener {
private AbsoluteTableIdentifier identifier;
private DataMapSchema dataMapSchema;
- private IndexDataMapFactory indexDataMapFactory;
+ private DataMapFactory dataMapFactory;
private BlockletDetailsFetcher blockletDetailsFetcher;
@@ -58,12 +64,12 @@ public final class TableDataMap extends OperationEventListener {
/**
* It is called to initialize and load the required table datamap metadata.
*/
- public TableDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema,
- IndexDataMapFactory indexDataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
+ TableDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema,
+ DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
this.dataMapSchema = dataMapSchema;
- this.indexDataMapFactory = indexDataMapFactory;
+ this.dataMapFactory = dataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
this.segmentPropertiesFetcher = segmentPropertiesFetcher;
}
@@ -85,9 +91,9 @@ public final class TableDataMap extends OperationEventListener {
if (filterExp == null) {
pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions);
} else {
- List<IndexDataMap> dataMaps = indexDataMapFactory.getDataMaps(segment);
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment);
- for (IndexDataMap dataMap : dataMaps) {
+ for (DataMap dataMap : dataMaps) {
pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
}
}
@@ -115,7 +121,7 @@ public final class TableDataMap extends OperationEventListener {
public List<DataMapDistributable> toDistributable(List<Segment> segments) throws IOException {
List<DataMapDistributable> distributables = new ArrayList<>();
for (Segment segment : segments) {
- List<DataMapDistributable> list = indexDataMapFactory.toDistributable(segment);
+ List<DataMapDistributable> list = dataMapFactory.toDistributable(segment);
for (DataMapDistributable distributable: list) {
distributable.setDataMapSchema(dataMapSchema);
distributable.setSegment(segment);
@@ -138,10 +144,10 @@ public final class TableDataMap extends OperationEventListener {
FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
List<Blocklet> blocklets = new ArrayList<>();
- List<IndexDataMap> indexDataMaps = indexDataMapFactory.getDataMaps(distributable);
- for (IndexDataMap indexDataMap : indexDataMaps) {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
+ for (DataMap dataMap : dataMaps) {
blocklets.addAll(
- indexDataMap.prune(
+ dataMap.prune(
filterExp,
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment()),
partitions));
@@ -150,13 +156,13 @@ public final class TableDataMap extends OperationEventListener {
String writePath =
identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapSchema
.getDataMapName();
- if (indexDataMapFactory.getDataMapType() == DataMapType.FG) {
+ if (dataMapFactory.getDataMapType() == DataMapLevel.FG) {
FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
}
for (Blocklet blocklet : blocklets) {
ExtendedBlocklet detailedBlocklet =
blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegment());
- if (indexDataMapFactory.getDataMapType() == DataMapType.FG) {
+ if (dataMapFactory.getDataMapType() == DataMapLevel.FG) {
String blockletwritePath =
writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
detailedBlocklet.setDataMapWriterPath(blockletwritePath);
@@ -174,7 +180,7 @@ public final class TableDataMap extends OperationEventListener {
*/
public void clear(List<Segment> segments) {
for (Segment segment: segments) {
- indexDataMapFactory.clear(segment);
+ dataMapFactory.clear(segment);
}
}
@@ -182,19 +188,19 @@ public final class TableDataMap extends OperationEventListener {
* Clears all datamap
*/
public void clear() {
- indexDataMapFactory.clear();
+ dataMapFactory.clear();
}
public DataMapSchema getDataMapSchema() {
return dataMapSchema;
}
- public IndexDataMapFactory getIndexDataMapFactory() {
- return indexDataMapFactory;
+ public DataMapFactory getDataMapFactory() {
+ return dataMapFactory;
}
@Override public void onEvent(Event event, OperationContext opContext) throws Exception {
- indexDataMapFactory.fireEvent(event);
+ dataMapFactory.fireEvent(event);
}
/**
@@ -209,8 +215,8 @@ public final class TableDataMap extends OperationEventListener {
throws IOException {
List<Segment> prunedSegments = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (Segment segment : segments) {
- List<IndexDataMap> dataMaps = indexDataMapFactory.getDataMaps(segment);
- for (IndexDataMap dataMap : dataMaps) {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
+ for (DataMap dataMap : dataMaps) {
if (dataMap.isScanRequired(filterExp)) {
// If any one task in a given segment contains the data that means the segment need to
// be scanned and we need to validate further data maps in the same segment
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
deleted file mode 100644
index c6961c7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-/**
- * Data Map writer
- */
-public abstract class AbstractDataMapWriter {
-
- protected AbsoluteTableIdentifier identifier;
-
- protected String segmentId;
-
- protected String writeDirectoryPath;
-
- public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, Segment segment,
- String writeDirectoryPath) {
- this.identifier = identifier;
- this.segmentId = segment.getSegmentNo();
- this.writeDirectoryPath = writeDirectoryPath;
- }
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- public abstract void onBlockStart(String blockId);
-
- /**
- * End of block notification
- */
- public abstract void onBlockEnd(String blockId);
-
- /**
- * Start of new blocklet notification.
- *
- * @param blockletId sequence number of blocklet in the block
- */
- public abstract void onBlockletStart(int blockletId);
-
- /**
- * End of blocklet notification
- *
- * @param blockletId sequence number of blocklet in the block
- */
- public abstract void onBlockletEnd(int blockletId);
-
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in IndexDataMapFactory.
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- public abstract void finish() throws IOException;
-
- /**
- * It copies the file from temp folder to actual folder
- *
- * @param dataMapFile
- * @throws IOException
- */
- protected void commitFile(String dataMapFile) throws IOException {
- if (!dataMapFile.startsWith(writeDirectoryPath)) {
- throw new UnsupportedOperationException(
- "Datamap file " + dataMapFile + " is not written in provided directory path "
- + writeDirectoryPath);
- }
- String dataMapFileName =
- dataMapFile.substring(writeDirectoryPath.length(), dataMapFile.length());
- String carbonFilePath = dataMapFileName.substring(0, dataMapFileName.lastIndexOf("/"));
- String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
- if (carbonFilePath.length() > 0) {
- carbonFilePath = segmentPath + carbonFilePath;
- FileFactory.mkdirs(carbonFilePath, FileFactory.getFileType(carbonFilePath));
- } else {
- carbonFilePath = segmentPath;
- }
- CarbonUtil.copyCarbonDataFileToCarbonStorePath(dataMapFile, carbonFilePath, 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
index 3d4c717..bd5f994 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/BlockletSerializer.java
@@ -20,9 +20,15 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+/**
+ * A serializer/deserializer for {@link FineGrainBlocklet}, it is used after prune the data
+ * by {@link org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainDataMap}
+ */
+@InterfaceAudience.Internal
public class BlockletSerializer {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
new file mode 100644
index 0000000..f036b0b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * An entity which can store and retrieve index data.
+ */
+@InterfaceAudience.Internal
+public interface DataMap<T extends Blocklet> {
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ void init(DataMapModel dataMapModel) throws MemoryException, IOException;
+
+ /**
+ * Prune the datamap with filter expression and partition information. It returns the list of
+ * blocklets where these filters can exist.
+ */
+ List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<PartitionSpec> partitions);
+
+ // TODO Move this method to Abstract class
+ /**
+ * Validate whether the current segment needs to be fetching the required data
+ */
+ boolean isScanRequired(FilterResolverIntf filterExp);
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ void clear();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
new file mode 100644
index 0000000..7fe910e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.events.Event;
+
+/**
+ * Interface for datamap factory, it is responsible for creating the datamap.
+ */
+public interface DataMapFactory<T extends DataMap> {
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
+
+ /**
+ * Return a new write for this datamap
+ */
+ DataMapWriter createWriter(Segment segment, String writeDirectoryPath);
+
+ /**
+ * Get the datamap for segmentid
+ */
+ List<T> getDataMaps(Segment segment) throws IOException;
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
+
+ /**
+ * Get all distributable objects of a segmentid
+ * @return
+ */
+ List<DataMapDistributable> toDistributable(Segment segment);
+
+ /**
+ *
+ * @param event
+ */
+ void fireEvent(Event event);
+
+ /**
+ * Clears datamap of the segment
+ */
+ void clear(Segment segment);
+
+ /**
+ * Clear all datamaps from memory
+ */
+ void clear();
+
+ /**
+ * Return metadata of this datamap
+ */
+ DataMapMeta getMeta();
+
+ /**
+ * Type of datamap whether it is FG or CG
+ */
+ DataMapLevel getDataMapType();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
new file mode 100644
index 0000000..4911f7b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+/**
+ * Writer interface for datamap.
+ * Developer should implement this interface to write index files.
+ * Writer will be called for every new block/blocklet/page is created when data load is executing.
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public abstract class DataMapWriter {
+
+ protected AbsoluteTableIdentifier identifier;
+
+ protected String segmentId;
+
+ protected String writeDirectoryPath;
+
+ public DataMapWriter(AbsoluteTableIdentifier identifier, Segment segment,
+ String writeDirectoryPath) {
+ this.identifier = identifier;
+ this.segmentId = segment.getSegmentNo();
+ this.writeDirectoryPath = writeDirectoryPath;
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ public abstract void onBlockStart(String blockId);
+
+ /**
+ * End of block notification
+ */
+ public abstract void onBlockEnd(String blockId);
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletStart(int blockletId);
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ public abstract void onBlockletEnd(int blockletId);
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ public abstract void finish() throws IOException;
+
+ /**
+ * It commits the index file by copying the file from temp folder to actual folder
+ *
+ * @param dataMapFile file path of index file
+ * @throws IOException if IO fails
+ */
+ protected void commitFile(String dataMapFile) throws IOException {
+ if (!dataMapFile.startsWith(writeDirectoryPath)) {
+ throw new UnsupportedOperationException(
+ "Datamap file " + dataMapFile + " is not written in provided directory path "
+ + writeDirectoryPath);
+ }
+ String dataMapFileName =
+ dataMapFile.substring(writeDirectoryPath.length(), dataMapFile.length());
+ String carbonFilePath = dataMapFileName.substring(0, dataMapFileName.lastIndexOf("/"));
+ String segmentPath = CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
+ if (carbonFilePath.length() > 0) {
+ carbonFilePath = segmentPath + carbonFilePath;
+ FileFactory.mkdirs(carbonFilePath, FileFactory.getFileType(carbonFilePath));
+ } else {
+ carbonFilePath = segmentPath;
+ }
+ CarbonUtil.copyCarbonDataFileToCarbonStorePath(dataMapFile, carbonFilePath, 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
deleted file mode 100644
index eed3ac9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-
-/**
- * Datamap is an entity which can store and retrieve index data.
- */
-public interface IndexDataMap<T extends Blocklet> {
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- void init(DataMapModel dataMapModel) throws MemoryException, IOException;
-
- /**
- * Prune the datamap with filter expression and partition information. It returns the list of
- * blocklets where these filters can exist.
- */
- List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions);
-
- // TODO Move this method to Abstract class
- /**
- * Validate whether the current segment needs to be fetching the required data
- */
- boolean isScanRequired(FilterResolverIntf filterExp);
-
- /**
- * Clear complete index table and release memory.
- */
- void clear();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
deleted file mode 100644
index 12c9fd9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.events.Event;
-
-/**
- * Interface for datamap factory, it is responsible for creating the datamap.
- */
-public interface IndexDataMapFactory<T extends IndexDataMap> {
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
-
- /**
- * Return a new write for this datamap
- */
- AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath);
-
- /**
- * Get the datamap for segmentid
- */
- List<T> getDataMaps(Segment segment) throws IOException;
-
- /**
- * Get datamaps for distributable object.
- */
- List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
-
- /**
- * Get all distributable objects of a segmentid
- * @return
- */
- List<DataMapDistributable> toDistributable(Segment segment);
-
- /**
- *
- * @param event
- */
- void fireEvent(Event event);
-
- /**
- * Clears datamap of the segment
- */
- void clear(Segment segment);
-
- /**
- * Clear all datamaps from memory
- */
- void clear();
-
- /**
- * Return metadata of this datamap
- */
- DataMapMeta getMeta();
-
- /**
- * Type of datamap whether it is FG or CG
- */
- DataMapType getDataMapType();
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
deleted file mode 100644
index df9d4e8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.cgdatamap;
-
-import org.apache.carbondata.core.datamap.dev.IndexDataMap;
-import org.apache.carbondata.core.indexstore.Blocklet;
-
-public abstract class AbstractCoarseGrainIndexDataMap implements IndexDataMap<Blocklet> {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
deleted file mode 100644
index 037c32e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.cgdatamap;
-
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
-
-/**
- * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
- * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
- * information of block and blocklet.
- * 3. Based on the splits scanrdd schedule the tasks.
- */
-public abstract class AbstractCoarseGrainIndexDataMapFactory
- implements IndexDataMapFactory<AbstractCoarseGrainIndexDataMap> {
-
- @Override public DataMapType getDataMapType() {
- return DataMapType.CG;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMap.java
new file mode 100644
index 0000000..62a1d1b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMap.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.indexstore.Blocklet;
+
+/**
+ * DataMap for Coarse Grain level, see {@link org.apache.carbondata.core.datamap.DataMapLevel#CG}
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public abstract class CoarseGrainDataMap implements DataMap<Blocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
new file mode 100644
index 0000000..4d20cdb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * Factory for {@link CoarseGrainDataMap}
+ * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
+ * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
+ * information of block and blocklet.
+ * 3. Based on the splits scanrdd schedule the tasks.
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public abstract class CoarseGrainDataMapFactory implements DataMapFactory<CoarseGrainDataMap> {
+
+ @Override public DataMapLevel getDataMapType() {
+ return DataMapLevel.CG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
index 2abe253..12b60b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -93,7 +93,7 @@ public class AndDataMapExprWrapper implements DataMapExprWrapper {
return wrappers;
}
- @Override public DataMapType getDataMapType() {
+ @Override public DataMapLevel getDataMapType() {
return left.getDataMapType();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
index 2baba6f..ddb19e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.List;
-import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -74,6 +74,6 @@ public interface DataMapExprWrapper extends Serializable {
* Get the datamap type.
* @return
*/
- DataMapType getDataMapType();
+ DataMapLevel getDataMapType();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
index 312fc11..d4be416 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
@@ -22,8 +22,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
@@ -82,7 +82,7 @@ public class DataMapExprWrapperImpl implements DataMapExprWrapper {
return wrappers;
}
- @Override public DataMapType getDataMapType() {
- return dataMap.getIndexDataMapFactory().getDataMapType();
+ @Override public DataMapLevel getDataMapType() {
+ return dataMap.getDataMapFactory().getDataMapType();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
index 3c63202..a8bfcad 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
@@ -22,8 +22,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -90,7 +90,7 @@ public class OrDataMapExprWrapper implements DataMapExprWrapper {
}
- @Override public DataMapType getDataMapType() {
+ @Override public DataMapLevel getDataMapType() {
return left.getDataMapType();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
deleted file mode 100644
index ea536b9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.fgdatamap;
-
-import org.apache.carbondata.core.datamap.dev.IndexDataMap;
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
-
-public abstract class AbstractFineGrainIndexDataMap implements IndexDataMap<FineGrainBlocklet> {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
deleted file mode 100644
index 762c233..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.fgdatamap;
-
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
-
-/**
- * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
- * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
- * information of block, blocklet, page and rowids information as well.
- * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
- * blocklet and filepath information as part of Splits.
- * 4. Based on the splits scanrdd schedule the tasks.
- * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
- * bitset if exists. And pass this bitset as input to it.
- */
-public abstract class AbstractFineGrainIndexDataMapFactory
- implements IndexDataMapFactory<AbstractFineGrainIndexDataMap> {
-
- @Override public DataMapType getDataMapType() {
- return DataMapType.FG;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainBlocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainBlocklet.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainBlocklet.java
new file mode 100644
index 0000000..b42500f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainBlocklet.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+/**
+ * Used for returning matched rows after pruned by {@link FineGrainDataMap}
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public class FineGrainBlocklet extends Blocklet implements Serializable {
+
+ private List<Page> pages;
+
+ public FineGrainBlocklet(String blockId, String blockletId, List<Page> pages) {
+ super(blockId, blockletId);
+ this.pages = pages;
+ }
+
+ // For serialization purpose
+ public FineGrainBlocklet() {
+
+ }
+
+ public List<Page> getPages() {
+ return pages;
+ }
+
+ public static class Page implements Writable,Serializable {
+
+ private int pageId;
+
+ private int[] rowId;
+
+ public BitSet getBitSet() {
+ BitSet bitSet =
+ new BitSet(CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
+ for (int row : rowId) {
+ bitSet.set(row);
+ }
+ return bitSet;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(pageId);
+ out.writeInt(rowId.length);
+ for (int i = 0; i < rowId.length; i++) {
+ out.writeInt(rowId[i]);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ pageId = in.readInt();
+ int length = in.readInt();
+ rowId = new int[length];
+ for (int i = 0; i < length; i++) {
+ rowId[i] = in.readInt();
+ }
+ }
+
+ public void setPageId(int pageId) {
+ this.pageId = pageId;
+ }
+
+ public void setRowId(int[] rowId) {
+ this.rowId = rowId;
+ }
+ }
+
+ public BitSetGroup getBitSetGroup(int numberOfPages) {
+ BitSetGroup bitSetGroup = new BitSetGroup(numberOfPages);
+ for (int i = 0; i < pages.size(); i++) {
+ bitSetGroup.setBitSet(pages.get(i).getBitSet(), pages.get(i).pageId);
+ }
+ return bitSetGroup;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ super.write(out);
+ int size = pages.size();
+ out.writeInt(size);
+ for (Page page : pages) {
+ page.write(out);
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ pages = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ Page page = new Page();
+ page.readFields(in);
+ pages.add(page);
+ }
+ }
+
+ @Override public boolean equals(Object o) {
+ return super.equals(o);
+ }
+
+ @Override public int hashCode() {
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMap.java
new file mode 100644
index 0000000..18389b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMap.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+
+/**
+ * DataMap for Fine Grain level, see {@link org.apache.carbondata.core.datamap.DataMapLevel#FG}
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public abstract class FineGrainDataMap implements DataMap<FineGrainBlocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMapFactory.java
new file mode 100644
index 0000000..0c9aaed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/FineGrainDataMapFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+
+/**
+ * Factory for {@link FineGrainDataMap}
+ *
+ * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
+ * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
+ * information of block, blocklet, page and rowids information as well.
+ * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
+ * blocklet and filepath information as part of Splits.
+ * 4. Based on the splits scanrdd schedule the tasks.
+ * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
+ * bitset if exists. And pass this bitset as input to it.
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Evolving
+public abstract class FineGrainDataMapFactory
+ implements DataMapFactory<FineGrainDataMap> {
+
+ @Override public DataMapLevel getDataMapType() {
+ return DataMapLevel.FG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 67dbe88..53ef496 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -31,8 +31,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapModel;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
@@ -47,7 +47,7 @@ import org.apache.hadoop.fs.RemoteIterator;
* blocks
*/
public class BlockletDataMapIndexStore
- implements Cache<TableBlockIndexUniqueIdentifier, BlockletIndexDataMap> {
+ implements Cache<TableBlockIndexUniqueIdentifier, BlockletDataMap> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMapIndexStore.class.getName());
/**
@@ -74,10 +74,10 @@ public class BlockletDataMapIndexStore
}
@Override
- public BlockletIndexDataMap get(TableBlockIndexUniqueIdentifier identifier)
+ public BlockletDataMap get(TableBlockIndexUniqueIdentifier identifier)
throws IOException {
String lruCacheKey = identifier.getUniqueTableSegmentIdentifier();
- BlockletIndexDataMap dataMap = (BlockletIndexDataMap) lruCache.get(lruCacheKey);
+ BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
if (dataMap == null) {
try {
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
@@ -134,15 +134,15 @@ public class BlockletDataMapIndexStore
}
@Override
- public List<BlockletIndexDataMap> getAll(
+ public List<BlockletDataMap> getAll(
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
- List<BlockletIndexDataMap> blockletDataMaps =
+ List<BlockletDataMap> blockletDataMaps =
new ArrayList<>(tableSegmentUniqueIdentifiers.size());
List<TableBlockIndexUniqueIdentifier> missedIdentifiers = new ArrayList<>();
// Get the datamaps for each indexfile from cache.
try {
for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
- BlockletIndexDataMap ifPresent = getIfPresent(identifier);
+ BlockletDataMap ifPresent = getIfPresent(identifier);
if (ifPresent != null) {
blockletDataMaps.add(ifPresent);
} else {
@@ -160,7 +160,7 @@ public class BlockletDataMapIndexStore
}
}
} catch (Throwable e) {
- for (BlockletIndexDataMap dataMap : blockletDataMaps) {
+ for (BlockletDataMap dataMap : blockletDataMaps) {
dataMap.clear();
}
throw new IOException("Problem in loading segment blocks.", e);
@@ -175,9 +175,9 @@ public class BlockletDataMapIndexStore
* @return
*/
@Override
- public BlockletIndexDataMap getIfPresent(
+ public BlockletDataMap getIfPresent(
TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
- return (BlockletIndexDataMap) lruCache.get(
+ return (BlockletDataMap) lruCache.get(
tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
}
@@ -200,7 +200,7 @@ public class BlockletDataMapIndexStore
* @return map of taks id to segment mapping
* @throws IOException
*/
- private BlockletIndexDataMap loadAndGetDataMap(
+ private BlockletDataMap loadAndGetDataMap(
TableBlockIndexUniqueIdentifier identifier,
SegmentIndexFileStore indexFileStore,
Map<String, BlockMetaInfo> blockMetaInfoMap)
@@ -211,9 +211,9 @@ public class BlockletDataMapIndexStore
if (lock == null) {
lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
}
- BlockletIndexDataMap dataMap;
+ BlockletDataMap dataMap;
synchronized (lock) {
- dataMap = new BlockletIndexDataMap();
+ dataMap = new BlockletDataMap();
dataMap.init(new BlockletDataMapModel(
identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()),
@@ -250,8 +250,8 @@ public class BlockletDataMapIndexStore
public void clearAccessCount(
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) {
for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
- BlockletIndexDataMap cacheable =
- (BlockletIndexDataMap) lruCache.get(identifier.getUniqueTableSegmentIdentifier());
+ BlockletDataMap cacheable =
+ (BlockletDataMap) lruCache.get(identifier.getUniqueTableSegmentIdentifier());
cacheable.clear();
}
}
[21/54] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad]
Add size base block allocation in data loading
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
index 7ea5cb3..e5583c2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java
@@ -19,20 +19,35 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.math.BigDecimal;
+import java.util.Arrays;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.memory.IntPointerBuffer;
import org.apache.carbondata.core.memory.MemoryBlock;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.memory.UnsafeSortMemoryManager;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.DataTypeUtil;
/**
* It can keep the data of prescribed size data in offheap/onheap memory and returns it when needed
*/
public class UnsafeCarbonRowPage {
+
+ private boolean[] noDictionaryDimensionMapping;
+
+ private boolean[] noDictionarySortColumnMapping;
+
+ private int dimensionSize;
+
+ private int measureSize;
+
+ private DataType[] measureDataType;
+
+ private long[] nullSetWords;
+
private IntPointerBuffer buffer;
private int lastSize;
@@ -47,14 +62,16 @@ public class UnsafeCarbonRowPage {
private long taskId;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
-
- public UnsafeCarbonRowPage(TableFieldStat tableFieldStat, MemoryBlock memoryBlock,
- boolean saveToDisk, long taskId) {
- this.tableFieldStat = tableFieldStat;
- this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+ public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
+ boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, DataType[] type,
+ MemoryBlock memoryBlock, boolean saveToDisk, long taskId) {
+ this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
+ this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
+ this.dimensionSize = dimensionSize;
+ this.measureSize = measureSize;
+ this.measureDataType = type;
this.saveToDisk = saveToDisk;
+ this.nullSetWords = new long[((measureSize - 1) >> 6) + 1];
this.taskId = taskId;
buffer = new IntPointerBuffer(this.taskId);
this.dataBlock = memoryBlock;
@@ -63,44 +80,255 @@ public class UnsafeCarbonRowPage {
this.managerType = MemoryManagerType.UNSAFE_MEMORY_MANAGER;
}
- public int addRow(Object[] row, ByteBuffer rowBuffer) {
- int size = addRow(row, dataBlock.getBaseOffset() + lastSize, rowBuffer);
+ public int addRow(Object[] row) {
+ int size = addRow(row, dataBlock.getBaseOffset() + lastSize);
buffer.set(lastSize);
lastSize = lastSize + size;
return size;
}
- /**
- * add raw row as intermidiate sort temp row to page
- *
- * @param row
- * @param address
- * @return
- */
- private int addRow(Object[] row, long address, ByteBuffer rowBuffer) {
- return sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToUnsafeMemory(row,
- dataBlock.getBaseObject(), address, rowBuffer);
+ private int addRow(Object[] row, long address) {
+ if (row == null) {
+ throw new RuntimeException("Row is null ??");
+ }
+ int dimCount = 0;
+ int size = 0;
+ Object baseObject = dataBlock.getBaseObject();
+ for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
+ if (noDictionaryDimensionMapping[dimCount]) {
+ byte[] col = (byte[]) row[dimCount];
+ CarbonUnsafe.getUnsafe()
+ .putShort(baseObject, address + size, (short) col.length);
+ size += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ address + size, col.length);
+ size += col.length;
+ } else {
+ int value = (int) row[dimCount];
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, value);
+ size += 4;
+ }
+ }
+
+ // write complex dimensions here.
+ for (; dimCount < dimensionSize; dimCount++) {
+ byte[] col = (byte[]) row[dimCount];
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) col.length);
+ size += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ address + size, col.length);
+ size += col.length;
+ }
+ Arrays.fill(nullSetWords, 0);
+ int nullSetSize = nullSetWords.length * 8;
+ int nullWordLoc = size;
+ size += nullSetSize;
+ for (int mesCount = 0; mesCount < measureSize; mesCount++) {
+ Object value = row[mesCount + dimensionSize];
+ if (null != value) {
+ DataType dataType = measureDataType[mesCount];
+ if (dataType == DataTypes.BOOLEAN) {
+ Boolean bval = (Boolean) value;
+ CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, bval);
+ size += 1;
+ } else if (dataType == DataTypes.SHORT) {
+ Short sval = (Short) value;
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
+ size += 2;
+ } else if (dataType == DataTypes.INT) {
+ Integer ival = (Integer) value;
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival);
+ size += 4;
+ } else if (dataType == DataTypes.LONG) {
+ Long val = (Long) value;
+ CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val);
+ size += 8;
+ } else if (dataType == DataTypes.DOUBLE) {
+ Double doubleVal = (Double) value;
+ CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal);
+ size += 8;
+ } else if (DataTypes.isDecimal(dataType)) {
+ BigDecimal decimalVal = (BigDecimal) value;
+ byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
+ CarbonUnsafe.getUnsafe()
+ .putShort(baseObject, address + size, (short) bigDecimalInBytes.length);
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
+ address + size, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
+ }
+ set(nullSetWords, mesCount);
+ } else {
+ unset(nullSetWords, mesCount);
+ }
+ }
+ CarbonUnsafe.getUnsafe().copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
+ address + nullWordLoc, nullSetSize);
+ return size;
}
- /**
- * get one row from memory address
- * @param address address
- * @return one row
- */
- public IntermediateSortTempRow getRow(long address) {
- return sortStepRowHandler.readIntermediateSortTempRowFromUnsafeMemory(
- dataBlock.getBaseObject(), address);
+ public Object[] getRow(long address, Object[] rowToFill) {
+ int dimCount = 0;
+ int size = 0;
+
+ Object baseObject = dataBlock.getBaseObject();
+ for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
+ if (noDictionaryDimensionMapping[dimCount]) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] col = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ col.length);
+ size += col.length;
+ rowToFill[dimCount] = col;
+ } else {
+ int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ rowToFill[dimCount] = anInt;
+ }
+ }
+
+ // write complex dimensions here.
+ for (; dimCount < dimensionSize; dimCount++) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] col = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
+ size += col.length;
+ rowToFill[dimCount] = col;
+ }
+
+ int nullSetSize = nullSetWords.length * 8;
+ Arrays.fill(nullSetWords, 0);
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
+ nullSetSize);
+ size += nullSetSize;
+
+ for (int mesCount = 0; mesCount < measureSize; mesCount++) {
+ if (isSet(nullSetWords, mesCount)) {
+ DataType dataType = measureDataType[mesCount];
+ if (dataType == DataTypes.BOOLEAN) {
+ Boolean bval = CarbonUnsafe.getUnsafe().getBoolean(baseObject, address + size);
+ size += 1;
+ rowToFill[dimensionSize + mesCount] = bval;
+ } else if (dataType == DataTypes.SHORT) {
+ Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ rowToFill[dimensionSize + mesCount] = sval;
+ } else if (dataType == DataTypes.INT) {
+ Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ rowToFill[dimensionSize + mesCount] = ival;
+ } else if (dataType == DataTypes.LONG) {
+ Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
+ size += 8;
+ rowToFill[dimensionSize + mesCount] = val;
+ } else if (dataType == DataTypes.DOUBLE) {
+ Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
+ size += 8;
+ rowToFill[dimensionSize + mesCount] = doubleVal;
+ } else if (DataTypes.isDecimal(dataType)) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] bigDecimalInBytes = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
+ }
+ } else {
+ rowToFill[dimensionSize + mesCount] = null;
+ }
+ }
+ return rowToFill;
}
- /**
- * write a row to stream
- * @param address address of a row
- * @param stream stream
- * @throws IOException
- */
- public void writeRow(long address, DataOutputStream stream) throws IOException {
- sortStepRowHandler.writeIntermediateSortTempRowFromUnsafeMemoryToStream(
- dataBlock.getBaseObject(), address, stream);
+ public void fillRow(long address, DataOutputStream stream) throws IOException {
+ int dimCount = 0;
+ int size = 0;
+
+ Object baseObject = dataBlock.getBaseObject();
+ for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
+ if (noDictionaryDimensionMapping[dimCount]) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] col = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ col.length);
+ size += col.length;
+ stream.writeShort(aShort);
+ stream.write(col);
+ } else {
+ int anInt = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ stream.writeInt(anInt);
+ }
+ }
+
+ // write complex dimensions here.
+ for (; dimCount < dimensionSize; dimCount++) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] col = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, col, CarbonUnsafe.BYTE_ARRAY_OFFSET, col.length);
+ size += col.length;
+ stream.writeShort(aShort);
+ stream.write(col);
+ }
+
+ int nullSetSize = nullSetWords.length * 8;
+ Arrays.fill(nullSetWords, 0);
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET,
+ nullSetSize);
+ size += nullSetSize;
+ for (int i = 0; i < nullSetWords.length; i++) {
+ stream.writeLong(nullSetWords[i]);
+ }
+
+ for (int mesCount = 0; mesCount < measureSize; mesCount++) {
+ if (isSet(nullSetWords, mesCount)) {
+ DataType dataType = measureDataType[mesCount];
+ if (dataType == DataTypes.SHORT) {
+ short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ size += 2;
+ stream.writeShort(sval);
+ } else if (dataType == DataTypes.INT) {
+ int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ size += 4;
+ stream.writeInt(ival);
+ } else if (dataType == DataTypes.LONG) {
+ long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
+ size += 8;
+ stream.writeLong(val);
+ } else if (dataType == DataTypes.DOUBLE) {
+ double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
+ size += 8;
+ stream.writeDouble(doubleVal);
+ } else if (DataTypes.isDecimal(dataType)) {
+ short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ byte[] bigDecimalInBytes = new byte[aShort];
+ size += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
+ size += bigDecimalInBytes.length;
+ stream.writeShort(aShort);
+ stream.write(bigDecimalInBytes);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
+ }
+ }
+ }
}
public void freeMemory() {
@@ -134,8 +362,27 @@ public class UnsafeCarbonRowPage {
return dataBlock;
}
- public TableFieldStat getTableFieldStat() {
- return tableFieldStat;
+ public static void set(long[] words, int index) {
+ int wordOffset = (index >> 6);
+ words[wordOffset] |= (1L << index);
+ }
+
+ public static void unset(long[] words, int index) {
+ int wordOffset = (index >> 6);
+ words[wordOffset] &= ~(1L << index);
+ }
+
+ public static boolean isSet(long[] words, int index) {
+ int wordOffset = (index >> 6);
+ return ((words[wordOffset] & (1L << index)) != 0);
+ }
+
+ public boolean[] getNoDictionaryDimensionMapping() {
+ return noDictionaryDimensionMapping;
+ }
+
+ public boolean[] getNoDictionarySortColumnMapping() {
+ return noDictionarySortColumnMapping;
}
public void setNewDataBlock(MemoryBlock newMemoryBlock) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index 5d038d3..4dd5e44 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.processing.loading.sort.unsafe;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -42,14 +41,13 @@ import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparator;
-import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDims;
+import org.apache.carbondata.processing.loading.sort.unsafe.comparator.UnsafeRowComparatorForNormalDIms;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort;
import org.apache.carbondata.processing.loading.sort.unsafe.sort.UnsafeIntSortDataFormat;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class UnsafeSortDataRows {
@@ -71,8 +69,7 @@ public class UnsafeSortDataRows {
*/
private SortParameters parameters;
- private TableFieldStat tableFieldStat;
- private ThreadLocal<ByteBuffer> rowBuffer;
+
private UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger;
private UnsafeCarbonRowPage rowPage;
@@ -97,13 +94,7 @@ public class UnsafeSortDataRows {
public UnsafeSortDataRows(SortParameters parameters,
UnsafeIntermediateMerger unsafeInMemoryIntermediateFileMerger, int inMemoryChunkSize) {
this.parameters = parameters;
- this.tableFieldStat = new TableFieldStat(parameters);
- this.rowBuffer = new ThreadLocal<ByteBuffer>() {
- @Override protected ByteBuffer initialValue() {
- byte[] backedArray = new byte[2 * 1024 * 1024];
- return ByteBuffer.wrap(backedArray);
- }
- };
+
this.unsafeInMemoryIntermediateFileMerger = unsafeInMemoryIntermediateFileMerger;
// observer of writing file in thread
@@ -136,7 +127,11 @@ public class UnsafeSortDataRows {
if (isMemoryAvailable) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size());
}
- this.rowPage = new UnsafeCarbonRowPage(tableFieldStat, baseBlock, !isMemoryAvailable, taskId);
+ this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
+ parameters.getDimColCount() + parameters.getComplexDimColCount(),
+ parameters.getMeasureColCount(), parameters.getMeasureDataType(), baseBlock,
+ !isMemoryAvailable, taskId);
// Delete if any older file exists in sort temp folder
deleteSortLocationIfExists();
@@ -183,7 +178,7 @@ public class UnsafeSortDataRows {
private void addBatch(Object[][] rowBatch, int size) throws CarbonSortKeyAndGroupByException {
for (int i = 0; i < size; i++) {
if (rowPage.canAdd()) {
- bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
+ bytesAdded += rowPage.addRow(rowBatch[i]);
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -199,8 +194,15 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
- bytesAdded += rowPage.addRow(rowBatch[i], rowBuffer.get());
+ rowPage = new UnsafeCarbonRowPage(
+ parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
+ parameters.getDimColCount() + parameters.getComplexDimColCount(),
+ parameters.getMeasureColCount(),
+ parameters.getMeasureDataType(),
+ memoryBlock,
+ saveToDisk, taskId);
+ bytesAdded += rowPage.addRow(rowBatch[i]);
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -218,7 +220,7 @@ public class UnsafeSortDataRows {
// if record holder list size is equal to sort buffer size then it will
// sort the list and then write current list data to file
if (rowPage.canAdd()) {
- rowPage.addRow(row, rowBuffer.get());
+ rowPage.addRow(row);
} else {
try {
if (enableInMemoryIntermediateMerge) {
@@ -233,8 +235,13 @@ public class UnsafeSortDataRows {
if (!saveToDisk) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(memoryBlock.size());
}
- rowPage = new UnsafeCarbonRowPage(tableFieldStat, memoryBlock, saveToDisk, taskId);
- rowPage.addRow(row, rowBuffer.get());
+ rowPage = new UnsafeCarbonRowPage(
+ parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
+ parameters.getDimColCount(), parameters.getMeasureColCount(),
+ parameters.getMeasureDataType(), memoryBlock,
+ saveToDisk, taskId);
+ rowPage.addRow(row);
} catch (Exception e) {
LOGGER.error(
"exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
@@ -262,7 +269,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(rowPage));
} else {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDims(rowPage));
+ new UnsafeRowComparatorForNormalDIms(rowPage));
}
unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
} else {
@@ -288,9 +295,10 @@ public class UnsafeSortDataRows {
// write number of entries to the file
stream.writeInt(actualSize);
for (int i = 0; i < actualSize; i++) {
- rowPage.writeRow(
- rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(), stream);
+ rowPage.fillRow(rowPage.getBuffer().get(i) + rowPage.getDataBlock().getBaseOffset(),
+ stream);
}
+
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
} finally {
@@ -359,7 +367,7 @@ public class UnsafeSortDataRows {
new UnsafeRowComparator(page));
} else {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDims(page));
+ new UnsafeRowComparatorForNormalDIms(page));
}
if (page.isSaveToDisk()) {
// create a new file every time
@@ -372,8 +380,7 @@ public class UnsafeSortDataRows {
writeDataToFile(page, sortTempFile);
LOGGER.info("Time taken to sort row page with size" + page.getBuffer().getActualSize()
+ " and write is: " + (System.currentTimeMillis() - startTime) + ": location:"
- + sortTempFile + ", sort temp file size in MB is "
- + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
+ + sortTempFile);
page.freeMemory();
// add sort temp filename to and arrayList. When the list size reaches 20 then
// intermediate merging of sort temp files will be triggered
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
index 33342dc..d02be9b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -23,25 +23,63 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
+
+ /**
+ * mapping of dictionary and no dictionary of sort_columns.
+ */
+ private boolean[] noDictionarySortColumnMaping;
+
private Object baseObject;
- private TableFieldStat tableFieldStat;
- private int dictSizeInMemory;
public UnsafeRowComparator(UnsafeCarbonRowPage rowPage) {
+ this.noDictionarySortColumnMaping = rowPage.getNoDictionarySortColumnMapping();
this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.tableFieldStat = rowPage.getTableFieldStat();
- this.dictSizeInMemory = (tableFieldStat.getDictSortDimCnt()
- + tableFieldStat.getDictNoSortDimCnt()) * 4;
}
/**
* Below method will be used to compare two mdkey
*/
public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- return compare(rowL, baseObject, rowR, baseObject);
+ int diff = 0;
+ long rowA = rowL.address;
+ long rowB = rowR.address;
+ int sizeA = 0;
+ int sizeB = 0;
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
+ if (isNoDictionary) {
+ short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowA + sizeA);
+ byte[] byteArr1 = new byte[aShort1];
+ sizeA += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowA + sizeA, byteArr1,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort1);
+ sizeA += aShort1;
+
+ short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObject, rowB + sizeB);
+ byte[] byteArr2 = new byte[aShort2];
+ sizeB += 2;
+ CarbonUnsafe.getUnsafe().copyMemory(baseObject, rowB + sizeB, byteArr2,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, aShort2);
+ sizeB += aShort2;
+
+ int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
+ if (difference != 0) {
+ return difference;
+ }
+ } else {
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
+ sizeA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
+ sizeB += 4;
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ }
+
+ return diff;
}
/**
@@ -52,40 +90,35 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
int diff = 0;
long rowA = rowL.address;
long rowB = rowR.address;
- int sizeInDictPartA = 0;
-
- int sizeInNonDictPartA = 0;
- int sizeInDictPartB = 0;
- int sizeInNonDictPartB = 0;
- for (boolean isNoDictionary : tableFieldStat.getIsSortColNoDictFlags()) {
+ int sizeA = 0;
+ int sizeB = 0;
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
- short lengthA = CarbonUnsafe.getUnsafe().getShort(baseObjectL,
- rowA + dictSizeInMemory + sizeInNonDictPartA);
- byte[] byteArr1 = new byte[lengthA];
- sizeInNonDictPartA += 2;
+ short aShort1 = CarbonUnsafe.getUnsafe().getShort(baseObjectL, rowA + sizeA);
+ byte[] byteArr1 = new byte[aShort1];
+ sizeA += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectL, rowA + dictSizeInMemory + sizeInNonDictPartA,
- byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthA);
- sizeInNonDictPartA += lengthA;
+ .copyMemory(baseObjectL, rowA + sizeA, byteArr1, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ aShort1);
+ sizeA += aShort1;
- short lengthB = CarbonUnsafe.getUnsafe().getShort(baseObjectR,
- rowB + dictSizeInMemory + sizeInNonDictPartB);
- byte[] byteArr2 = new byte[lengthB];
- sizeInNonDictPartB += 2;
+ short aShort2 = CarbonUnsafe.getUnsafe().getShort(baseObjectR, rowB + sizeB);
+ byte[] byteArr2 = new byte[aShort2];
+ sizeB += 2;
CarbonUnsafe.getUnsafe()
- .copyMemory(baseObjectR, rowB + dictSizeInMemory + sizeInNonDictPartB,
- byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET, lengthB);
- sizeInNonDictPartB += lengthB;
+ .copyMemory(baseObjectR, rowB + sizeB, byteArr2, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ aShort2);
+ sizeB += aShort2;
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
if (difference != 0) {
return difference;
}
} else {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeInDictPartA);
- sizeInDictPartA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeInDictPartB);
- sizeInDictPartB += 4;
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObjectL, rowA + sizeA);
+ sizeA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObjectR, rowB + sizeB);
+ sizeB += 4;
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
new file mode 100644
index 0000000..483dcb2
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
+
+public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbonRow> {
+
+ private Object baseObject;
+
+ private int numberOfSortColumns;
+
+ public UnsafeRowComparatorForNormalDIms(UnsafeCarbonRowPage rowPage) {
+ this.baseObject = rowPage.getDataBlock().getBaseObject();
+ this.numberOfSortColumns = rowPage.getNoDictionarySortColumnMapping().length;
+ }
+
+ /**
+ * Below method will be used to compare two mdkey
+ */
+ public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
+ int diff = 0;
+ long rowA = rowL.address;
+ long rowB = rowR.address;
+ int sizeA = 0;
+ int sizeB = 0;
+ for (int i = 0; i < numberOfSortColumns; i++) {
+ int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
+ sizeA += 4;
+ int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
+ sizeB += 4;
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
deleted file mode 100644
index e9cfb1c..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/comparator/UnsafeRowComparatorForNormalDims.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort.unsafe.comparator;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeCarbonRow;
-
-public class UnsafeRowComparatorForNormalDims implements Comparator<UnsafeCarbonRow> {
-
- private Object baseObject;
-
- private int numberOfSortColumns;
-
- public UnsafeRowComparatorForNormalDims(UnsafeCarbonRowPage rowPage) {
- this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.numberOfSortColumns = rowPage.getTableFieldStat().getIsSortColNoDictFlags().length;
- }
-
- /**
- * Below method will be used to compare two mdkey
- */
- public int compare(UnsafeCarbonRow rowL, UnsafeCarbonRow rowR) {
- int diff = 0;
- long rowA = rowL.address;
- long rowB = rowR.address;
- int sizeA = 0;
- int sizeB = 0;
- for (int i = 0; i < numberOfSortColumns; i++) {
- int dimFieldA = CarbonUnsafe.getUnsafe().getInt(baseObject, rowA + sizeA);
- sizeA += 4;
- int dimFieldB = CarbonUnsafe.getUnsafe().getInt(baseObject, rowB + sizeB);
- sizeB += 4;
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
-
- return diff;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
index d790c41..686e855 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/SortTempChunkHolder.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.processing.loading.sort.unsafe.holder;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
/**
@@ -29,7 +28,7 @@ public interface SortTempChunkHolder extends Comparable<SortTempChunkHolder> {
void readRow() throws CarbonSortKeyAndGroupByException;
- IntermediateSortTempRow getRow();
+ Object[] getRow();
int numberOfRows();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index a776db1..6b0cfa6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -19,10 +19,9 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeInMemoryIntermediateDataMerger;
-import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
@@ -39,18 +38,21 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage[] rowPages;
- private IntermediateSortTempRowComparator comparator;
+ private NewRowComparator comparator;
- private IntermediateSortTempRow currentRow;
+ private Object[] currentRow;
+
+ private int columnSize;
public UnsafeFinalMergePageHolder(UnsafeInMemoryIntermediateDataMerger merger,
- boolean[] noDictSortColumnMapping) {
+ boolean[] noDictSortColumnMapping, int columnSize) {
this.actualSize = merger.getEntryCount();
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new IntermediateSortTempRowComparator(noDictSortColumnMapping);
+ this.comparator = new NewRowComparator(noDictSortColumnMapping);
+ this.columnSize = columnSize;
}
public boolean hasNext() {
@@ -61,11 +63,12 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
}
public void readRow() {
- currentRow = rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter]);
+ currentRow = new Object[columnSize];
+ rowPages[rowPageIndexes[counter]].getRow(mergedAddresses[counter], currentRow);
counter++;
}
- public IntermediateSortTempRow getRow() {
+ public Object[] getRow() {
return currentRow;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index cbcbbae..6f05088 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -19,9 +19,8 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
public class UnsafeInmemoryHolder implements SortTempChunkHolder {
@@ -34,18 +33,21 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage rowPage;
- private IntermediateSortTempRow currentRow;
+ private Object[] currentRow;
private long address;
- private IntermediateSortTempRowComparator comparator;
+ private NewRowComparator comparator;
- public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage) {
+ private int columnSize;
+
+ public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize,
+ int numberOfSortColumns) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new IntermediateSortTempRowComparator(
- rowPage.getTableFieldStat().getIsSortColNoDictFlags());
+ this.comparator = new NewRowComparator(rowPage.getNoDictionarySortColumnMapping());
+ this.columnSize = columnSize;
}
public boolean hasNext() {
@@ -56,12 +58,13 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
}
public void readRow() {
+ currentRow = new Object[columnSize];
address = rowPage.getBuffer().get(counter);
- currentRow = rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset());
+ rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset(), currentRow);
counter++;
}
- public IntermediateSortTempRow getRow() {
+ public Object[] getRow() {
return currentRow;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 527452a..11b3d43 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,14 +31,15 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -62,15 +63,21 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
* entry count
*/
private int entryCount;
+
/**
* return row
*/
- private IntermediateSortTempRow returnRow;
+ private Object[] returnRow;
+ private int dimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private DataType[] measureDataTypes;
private int readBufferSize;
private String compressorName;
- private IntermediateSortTempRow[] currentBuffer;
+ private Object[][] currentBuffer;
- private IntermediateSortTempRow[] backupBuffer;
+ private Object[][] backupBuffer;
private boolean isBackupFilled;
@@ -93,21 +100,27 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
private int numberOfObjectRead;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
- private Comparator<IntermediateSortTempRow> comparator;
+ private int nullSetWordsLength;
+
+ private Comparator<Object[]> comparator;
+
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters parameters) {
// set temp file
this.tempFile = tempFile;
+ this.dimCnt = parameters.getDimColCount();
+ this.complexCnt = parameters.getComplexDimColCount();
+ this.measureCnt = parameters.getMeasureColCount();
+ this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
+ this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
- this.tableFieldStat = new TableFieldStat(parameters);
- this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+
this.executorService = Executors.newFixedThreadPool(1);
- comparator = new IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
+ this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
+ comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -156,17 +169,11 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
- @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- try {
- this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
- this.numberOfObjectRead++;
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException("Problems while reading row", e);
- }
+ this.returnRow = getRowFromStream();
}
}
@@ -200,22 +207,63 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
}
/**
- * get a batch of row, this interface is used in reading compressed sort temp files
- *
- * @param expected expected number in a batch
- * @return a batch of row
- * @throws IOException if error occurs while reading from stream
+ * @return
+ * @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow[] readBatchedRowFromStream(int expected)
- throws IOException {
- IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
- for (int i = 0; i < expected; i++) {
- IntermediateSortTempRow holder
- = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
- holders[i] = holder;
+ private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
+ Object[] row = new Object[dimCnt + measureCnt];
+ try {
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ short aShort = stream.readShort();
+ byte[] col = new byte[aShort];
+ stream.readFully(col);
+ row[dimCount] = col;
+ } else {
+ int anInt = stream.readInt();
+ row[dimCount] = anInt;
+ }
+ }
+
+ // write complex dimensions here.
+ for (; dimCount < dimCnt; dimCount++) {
+ short aShort = stream.readShort();
+ byte[] col = new byte[aShort];
+ stream.readFully(col);
+ row[dimCount] = col;
+ }
+
+ long[] words = new long[nullSetWordsLength];
+ for (int i = 0; i < words.length; i++) {
+ words[i] = stream.readLong();
+ }
+
+ for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
+ if (UnsafeCarbonRowPage.isSet(words, mesCount)) {
+ DataType dataType = measureDataTypes[mesCount];
+ if (dataType == DataTypes.SHORT) {
+ row[dimCount + mesCount] = stream.readShort();
+ } else if (dataType == DataTypes.INT) {
+ row[dimCount + mesCount] = stream.readInt();
+ } else if (dataType == DataTypes.LONG) {
+ row[dimCount + mesCount] = stream.readLong();
+ } else if (dataType == DataTypes.DOUBLE) {
+ row[dimCount + mesCount] = stream.readDouble();
+ } else if (DataTypes.isDecimal(dataType)) {
+ short aShort = stream.readShort();
+ byte[] bigDecimalInBytes = new byte[aShort];
+ stream.readFully(bigDecimalInBytes);
+ row[dimCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + dataType);
+ }
+ }
+ }
+ return row;
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException(e);
}
- this.numberOfObjectRead += expected;
- return holders;
}
/**
@@ -223,7 +271,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @return row
*/
- public IntermediateSortTempRow getRow() {
+ public Object[] getRow() {
return this.returnRow;
}
@@ -278,7 +326,9 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@Override public int hashCode() {
int hash = 0;
- hash += tableFieldStat.hashCode();
+ hash += 31 * measureCnt;
+ hash += 31 * dimCnt;
+ hash += 31 * complexCnt;
hash += tempFile.hashCode();
return hash;
}
@@ -318,12 +368,16 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords number of records to be read
- * @return batch of intermediate sort temp row
- * @throws IOException if error occurs reading records from file
+ * @param numberOfRecords
+ * @return
+ * @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
- throws IOException {
- return readBatchedRowFromStream(numberOfRecords);
+ private Object[][] prefetchRecordsFromFile(int numberOfRecords)
+ throws CarbonSortKeyAndGroupByException {
+ Object[][] records = new Object[numberOfRecords][];
+ for (int i = 0; i < numberOfRecords; i++) {
+ records[i] = getRowFromStream();
+ }
+ return records;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index 22673ff..4bbf61b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -21,21 +21,25 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.AbstractQueue;
+import java.util.Arrays;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeSortTempFileChunkHolder;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
@@ -65,13 +69,22 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
private int totalNumberOfRecords;
private SortParameters mergerParameters;
- private TableFieldStat tableFieldStat;
+
private File[] intermediateFiles;
+
private File outPutFile;
+ private int dimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
- private SortStepRowHandler sortStepRowHandler;
+
+ private long[] nullSetWords;
+
+ private ByteBuffer rowData;
private Throwable throwable;
@@ -84,10 +97,16 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
+ this.dimCnt = mergerParameters.getDimColCount();
+ this.complexCnt = mergerParameters.getComplexDimColCount();
+ this.measureCnt = mergerParameters.getMeasureColCount();
+ this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
+ this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
- this.tableFieldStat = new TableFieldStat(mergerParameters);
- this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+ this.nullSetWords = new long[((measureCnt - 1) >> 6) + 1];
+ // Take size of 2 MB for each row. I think it is high enough to use
+ rowData = ByteBuffer.allocate(2 * 1024 * 1024);
}
@Override public Void call() throws Exception {
@@ -146,14 +165,13 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get sorted sort temp row from the sort temp files
+ * This method will be used to get the sorted record from file
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow getSortedRecordFromFile()
- throws CarbonSortKeyAndGroupByException {
- IntermediateSortTempRow row = null;
+ private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
+ Object[] row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -217,7 +235,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size" + this.recordHolderHeap.size());
}
/**
@@ -232,12 +250,12 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted sort temp row
+ * This method will be used to get the sorted row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
+ private Object[] next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -254,16 +272,82 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws IOException problem while writing
+ * @throws CarbonSortKeyAndGroupByException problem while writing
*/
- private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
- sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
+ private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException, IOException {
+ int dimCount = 0;
+ int size = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ byte[] col = (byte[]) row[dimCount];
+ rowData.putShort((short) col.length);
+ size += 2;
+ rowData.put(col);
+ size += col.length;
+ } else {
+ rowData.putInt((int) row[dimCount]);
+ size += 4;
+ }
+ }
+
+ // write complex dimensions here.
+ int dimensionSize = dimCnt + complexCnt;
+ for (; dimCount < dimensionSize; dimCount++) {
+ byte[] col = (byte[]) row[dimCount];
+ rowData.putShort((short)col.length);
+ size += 2;
+ rowData.put(col);
+ size += col.length;
+ }
+ Arrays.fill(nullSetWords, 0);
+ int nullSetSize = nullSetWords.length * 8;
+ int nullLoc = size;
+ size += nullSetSize;
+ for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
+ Object value = row[mesCount + dimensionSize];
+ if (null != value) {
+ DataType dataType = measureDataTypes[mesCount];
+ if (dataType == DataTypes.SHORT) {
+ rowData.putShort(size, (Short) value);
+ size += 2;
+ } else if (dataType == DataTypes.INT) {
+ rowData.putInt(size, (Integer) value);
+ size += 4;
+ } else if (dataType == DataTypes.LONG) {
+ rowData.putLong(size, (Long) value);
+ size += 8;
+ } else if (dataType == DataTypes.DOUBLE) {
+ rowData.putDouble(size, (Double) value);
+ size += 8;
+ } else if (DataTypes.isDecimal(dataType)) {
+ byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(((BigDecimal) value));
+ rowData.putShort(size, (short) bigDecimalInBytes.length);
+ size += 2;
+ for (int i = 0; i < bigDecimalInBytes.length; i++) {
+ rowData.put(size++, bigDecimalInBytes[i]);
+ }
+ }
+ UnsafeCarbonRowPage.set(nullSetWords, mesCount);
+ } else {
+ UnsafeCarbonRowPage.unset(nullSetWords, mesCount);
+ }
+ }
+ for (int i = 0; i < nullSetWords.length; i++) {
+ rowData.putLong(nullLoc, nullSetWords[i]);
+ nullLoc += 8;
+ }
+ byte[] rowBytes = new byte[size];
+ rowData.position(0);
+ rowData.get(rowBytes);
+ stream.write(rowBytes);
+ rowData.clear();
}
private void finish() throws CarbonSortKeyAndGroupByException {
clear();
try {
CarbonUtil.deleteFiles(intermediateFiles);
+ rowData.clear();
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate files");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index 64f3c25..ce118d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -29,8 +29,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.processing.loading.sort.SortStepRowUtil;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder;
@@ -56,7 +55,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
private AbstractQueue<SortTempChunkHolder> recordHolderHeapLocal;
private SortParameters parameters;
- private SortStepRowHandler sortStepRowHandler;
+ private SortStepRowUtil sortStepRowUtil;
/**
* tempFileLocation
*/
@@ -69,7 +68,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
public UnsafeSingleThreadFinalSortFilesMerger(SortParameters parameters,
String[] tempFileLocation) {
this.parameters = parameters;
- this.sortStepRowHandler = new SortStepRowHandler(parameters);
+ this.sortStepRowUtil = new SortStepRowUtil(parameters);
this.tempFileLocation = tempFileLocation;
this.tableName = parameters.getTableName();
}
@@ -109,7 +108,9 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
LOGGER.info("Started adding first record from each page");
for (final UnsafeCarbonRowPage rowPage : rowPages) {
- SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
+ SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage,
+ parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
+ .getMeasureColCount(), parameters.getNumberOfSortColumns());
// initialize
sortTempFileChunkHolder.readRow();
@@ -120,7 +121,9 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder =
- new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn());
+ new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(),
+ parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
+ .getMeasureColCount());
// initialize
sortTempFileChunkHolder.readRow();
@@ -139,7 +142,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
+ LOGGER.info("Heap Size" + this.recordHolderHeapLocal.size());
} catch (Exception e) {
LOGGER.error(e);
throw new CarbonDataWriterException(e);
@@ -177,14 +180,12 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
}
/**
- * This method will be used to get the sorted row in 3-parted format.
- * The row will feed the following writer process step.
+ * This method will be used to get the sorted row
*
* @return sorted row
*/
public Object[] next() {
- IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
- return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
+ return sortStepRowUtil.convertRow(getSortedRecordFromFile());
}
/**
@@ -192,8 +193,8 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
*
* @return sorted record sorted record
*/
- private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
- IntermediateSortTempRow row = null;
+ private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
+ Object[] row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 4579c85..1ab803b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -1005,14 +1005,8 @@ public final class CarbonDataMergerUtil {
/**
* This method traverses Update Delta Files inside the seg and return true
* if UpdateDelta Files are more than IUD Compaction threshold.
- *
- * @param seg
- * @param identifier
- * @param segmentUpdateStatusManager
- * @param numberDeltaFilesThreshold
- * @return
*/
- public static Boolean checkUpdateDeltaFilesInSeg(Segment seg,
+ private static Boolean checkUpdateDeltaFilesInSeg(Segment seg,
AbsoluteTableIdentifier identifier, SegmentUpdateStatusManager segmentUpdateStatusManager,
int numberDeltaFilesThreshold) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index bce1b33..038d34c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -389,6 +389,7 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
noDictionarySortColumnMapping, 0, noDictionarySortColumnMapping.length);
}
sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
+
String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
finalMerger =
@@ -406,8 +407,8 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
+ carbonLoadModel.getFactTimeStamp() + ".tmp";
} else {
carbonStoreLocation = CarbonDataProcessorUtil
- .createCarbonStoreLocation(carbonTable.getTablePath(), carbonLoadModel.getDatabaseName(),
- tableName, carbonLoadModel.getSegmentId());
+ .createCarbonStoreLocation(carbonLoadModel.getDatabaseName(), tableName,
+ carbonLoadModel.getSegmentId());
}
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index 4aca13a..2616def 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -76,9 +76,8 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + loadModel
.getFactTimeStamp() + ".tmp";
} else {
- carbonStoreLocation = CarbonDataProcessorUtil
- .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getSegmentId());
+ carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation(
+ loadModel.getDatabaseName(), tableName, loadModel.getSegmentId());
}
CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel
.getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
index 92db4c5..221697f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java
@@ -47,9 +47,8 @@ public class RowResultProcessor {
CarbonDataProcessorUtil.createLocations(tempStoreLocation);
this.segmentProperties = segProp;
String tableName = carbonTable.getTableName();
- String carbonStoreLocation = CarbonDataProcessorUtil
- .createCarbonStoreLocation(carbonTable.getTablePath(), loadModel.getDatabaseName(),
- tableName, loadModel.getSegmentId());
+ String carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation(
+ loadModel.getDatabaseName(), tableName, loadModel.getSegmentId());
CarbonFactDataHandlerModel carbonFactDataHandlerModel =
CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable,
segProp, tableName, tempStoreLocation, carbonStoreLocation);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index c06819c..04efa1f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.math.BigDecimal;
import java.util.AbstractQueue;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
@@ -28,9 +29,11 @@ import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.NonDictionaryUtil;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class IntermediateFileMerger implements Callable<Void> {
@@ -65,12 +68,17 @@ public class IntermediateFileMerger implements Callable<Void> {
private File[] intermediateFiles;
private File outPutFile;
+ private int dimCnt;
+ private int noDictDimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
private Throwable throwable;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
+
/**
* IntermediateFileMerger Constructor
*/
@@ -80,10 +88,14 @@ public class IntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
+ this.dimCnt = mergerParameters.getDimColCount();
+ this.noDictDimCnt = mergerParameters.getNoDictionaryCount();
+ this.complexCnt = mergerParameters.getComplexDimColCount();
+ this.measureCnt = mergerParameters.getMeasureColCount();
+ this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
+ this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
- this.tableFieldStat = new TableFieldStat(mergerParameters);
- this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
}
@Override public Void call() throws Exception {
@@ -142,14 +154,13 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted sort temp row from sort temp file
+ * This method will be used to get the sorted record from file
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow getSortedRecordFromFile()
- throws CarbonSortKeyAndGroupByException {
- IntermediateSortTempRow row = null;
+ private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
+ Object[] row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -216,7 +227,7 @@ public class IntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size" + this.recordHolderHeap.size());
}
/**
@@ -231,12 +242,12 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted sort temp row
+ * This method will be used to get the sorted row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
+ private Object[] next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -253,10 +264,62 @@ public class IntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws IOException problem while writing
+ * @throws CarbonSortKeyAndGroupByException problem while writing
*/
- private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
- sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
+ private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException {
+ try {
+ int[] mdkArray = (int[]) row[0];
+ byte[][] nonDictArray = (byte[][]) row[1];
+ int mdkIndex = 0;
+ int nonDictKeyIndex = 0;
+ // write dictionary and non dictionary dimensions here.
+ for (boolean nodictinary : isNoDictionaryDimensionColumn) {
+ if (nodictinary) {
+ byte[] col = nonDictArray[nonDictKeyIndex++];
+ stream.writeShort(col.length);
+ stream.write(col);
+ } else {
+ stream.writeInt(mdkArray[mdkIndex++]);
+ }
+ }
+ // write complex
+ for (; nonDictKeyIndex < noDictDimCnt + complexCnt; nonDictKeyIndex++) {
+ byte[] col = nonDictArray[nonDictKeyIndex++];
+ stream.writeShort(col.length);
+ stream.write(col);
+ }
+ // write measure
+ int fieldIndex = 0;
+ for (int counter = 0; counter < measureCnt; counter++) {
+ if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
+ stream.write((byte) 1);
+ DataType dataType = measureDataTypes[counter];
+ if (dataType == DataTypes.BOOLEAN) {
+ stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (dataType == DataTypes.SHORT) {
+ stream.writeShort((short) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (dataType == DataTypes.INT) {
+ stream.writeInt((int) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (dataType == DataTypes.LONG) {
+ stream.writeLong((long) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (dataType == DataTypes.DOUBLE) {
+ stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ } else if (DataTypes.isDecimal(dataType)) {
+ byte[] bigDecimalInBytes = DataTypeUtil
+ .bigDecimalToByte((BigDecimal) NonDictionaryUtil.getMeasure(fieldIndex, row));
+ stream.writeInt(bigDecimalInBytes.length);
+ stream.write(bigDecimalInBytes);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + dataType);
+ }
+ } else {
+ stream.write((byte) 0);
+ }
+ fieldIndex++;
+ }
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
+ }
}
private void finish() throws CarbonSortKeyAndGroupByException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
deleted file mode 100644
index 9b6d1e8..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-
-/**
- * This class is used as comparator for comparing intermediate sort temp row
- */
-public class IntermediateSortTempRowComparator implements Comparator<IntermediateSortTempRow> {
- /**
- * isSortColumnNoDictionary whether the sort column is not dictionary or not
- */
- private boolean[] isSortColumnNoDictionary;
-
- /**
- * @param isSortColumnNoDictionary isSortColumnNoDictionary
- */
- public IntermediateSortTempRowComparator(boolean[] isSortColumnNoDictionary) {
- this.isSortColumnNoDictionary = isSortColumnNoDictionary;
- }
-
- /**
- * Below method will be used to compare two sort temp row
- */
- public int compare(IntermediateSortTempRow rowA, IntermediateSortTempRow rowB) {
- int diff = 0;
- int dictIndex = 0;
- int nonDictIndex = 0;
-
- for (boolean isNoDictionary : isSortColumnNoDictionary) {
-
- if (isNoDictionary) {
- byte[] byteArr1 = rowA.getNoDictSortDims()[nonDictIndex];
- byte[] byteArr2 = rowB.getNoDictSortDims()[nonDictIndex];
- nonDictIndex++;
-
- int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
- if (difference != 0) {
- return difference;
- }
- } else {
- int dimFieldA = rowA.getDictSortDims()[dictIndex];
- int dimFieldB = rowB.getDictSortDims()[dictIndex];
- dictIndex++;
-
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
- }
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
index 3f94533..d2579d2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
@@ -40,11 +40,14 @@ public class NewRowComparator implements Comparator<Object[]> {
*/
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
+
int index = 0;
for (boolean isNoDictionary : noDictionarySortColumnMaping) {
+
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[index];
+
byte[] byteArr2 = (byte[]) rowB[index];
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
@@ -54,7 +57,6 @@ public class NewRowComparator implements Comparator<Object[]> {
} else {
int dimFieldA = (int) rowA[index];
int dimFieldB = (int) rowB[index];
-
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
@@ -63,6 +65,7 @@ public class NewRowComparator implements Comparator<Object[]> {
index++;
}
+
return diff;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
index 7538c92..e01b587 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
@@ -29,7 +29,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
private int numberOfSortColumns;
/**
- * NewRowComparatorForNormalDims Constructor
+ * RowComparatorForNormalDims Constructor
*
* @param numberOfSortColumns
*/
@@ -46,6 +46,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
int diff = 0;
for (int i = 0; i < numberOfSortColumns; i++) {
+
int dimFieldA = (int)rowA[i];
int dimFieldB = (int)rowB[i];
diff = dimFieldA - dimFieldB;
[14/54] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad]
Add size base block allocation in data loading
Posted by ra...@apache.org.
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9a423c24
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9a423c24
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9a423c24
Branch: refs/heads/master
Commit: 9a423c241b01d6e0b3f1946c052230adb47f4538
Parents: 2b41f14
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../constants/CarbonLoadOptionConstants.java | 10 +
.../core/datastore/block/TableBlockInfo.java | 29 ++
.../carbondata/core/util/CarbonProperties.java | 11 +
docs/useful-tips-on-carbondata.md | 1 +
.../spark/rdd/NewCarbonDataLoadRDD.scala | 4 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 18 +-
.../merger/NodeMultiBlockRelation.java | 40 ++
.../processing/util/CarbonLoaderUtil.java | 480 ++++++++++++-------
.../processing/util/CarbonLoaderUtilTest.java | 125 +++++
10 files changed, 545 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index bcfeba0..a6bf60f 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -114,4 +114,14 @@ public final class CarbonLoadOptionConstants {
*/
public static final int MAX_EXTERNAL_DICTIONARY_SIZE = 10000000;
+ /**
+ * enable block size based block allocation while loading data. By default, carbondata assigns
+ * blocks to node based on block number. If this option is set to `true`, carbondata will
+ * consider block size first and make sure that all the nodes will process almost equal size of
+ * data. This option is especially useful when you encounter skewed data.
+ */
+ @CarbonProperty
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
+ = "carbon.load.skewedDataOptimization.enabled";
+ public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a7bfdba..c0cebe0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.core.datastore.block;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@@ -98,6 +100,20 @@ public class TableBlockInfo implements Distributable, Serializable {
private String dataMapWriterPath;
+ /**
+ * comparator to sort by block size in descending order.
+ * Since each line is not exactly the same, the size of a InputSplit may differs,
+ * so we allow some deviation for these splits.
+ */
+ public static final Comparator<Distributable> DATA_SIZE_DESC_COMPARATOR =
+ new Comparator<Distributable>() {
+ @Override public int compare(Distributable o1, Distributable o2) {
+ long diff =
+ ((TableBlockInfo) o1).getBlockLength() - ((TableBlockInfo) o2).getBlockLength();
+ return diff < 0 ? 1 : (diff == 0 ? 0 : -1);
+ }
+ };
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId,
String[] locations, long blockLength, ColumnarFormatVersion version,
String[] deletedDeltaFilePath) {
@@ -434,4 +450,17 @@ public class TableBlockInfo implements Distributable, Serializable {
public void setDataMapWriterPath(String dataMapWriterPath) {
this.dataMapWriterPath = dataMapWriterPath;
}
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("TableBlockInfo{");
+ sb.append("filePath='").append(filePath).append('\'');
+ sb.append(", blockOffset=").append(blockOffset);
+ sb.append(", blockLength=").append(blockLength);
+ sb.append(", segmentId='").append(segmentId).append('\'');
+ sb.append(", blockletId='").append(blockletId).append('\'');
+ sb.append(", locations=").append(Arrays.toString(locations));
+ sb.append('}');
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 8b81c1e..6fd087b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1235,6 +1235,17 @@ public final class CarbonProperties {
return CarbonCommonConstants.CARBON_SORT_TEMP_COMPRESSOR_DEFAULT;
}
}
+
+ /**
+ * whether optimization for skewed data is enabled
+ * @return true, if enabled; false for not enabled.
+ */
+ public boolean isLoadSkewedDataOptimizationEnabled() {
+ String skewedEnabled = getProperty(
+ CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION,
+ CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT);
+ return skewedEnabled.equalsIgnoreCase("true");
+ }
/**
* returns true if carbon property
* @param key
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/docs/useful-tips-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/useful-tips-on-carbondata.md b/docs/useful-tips-on-carbondata.md
index 4d43003..ff339d0 100644
--- a/docs/useful-tips-on-carbondata.md
+++ b/docs/useful-tips-on-carbondata.md
@@ -169,5 +169,6 @@
| carbon.use.local.dir | spark/carbonlib/carbon.properties | Data loading | Whether use YARN local directories for multi-table load disk load balance | If this is set it to true CarbonData will use YARN local directories for multi-table load disk load balance, that will improve the data load performance. |
| carbon.use.multiple.temp.dir | spark/carbonlib/carbon.properties | Data loading | Whether to use multiple YARN local directories during table data loading for disk load balance | After enabling 'carbon.use.local.dir', if this is set to true, CarbonData will use all YARN local directories during data load for disk load balance, that will improve the data load performance. Please enable this property when you encounter disk hotspot problem during data loading. |
| carbon.sort.temp.compressor | spark/carbonlib/carbon.properties | Data loading | Specify the name of compressor to compress the intermediate sort temporary files during sort procedure in data loading. | The optional values are 'SNAPPY','GZIP','BZIP2','LZ4' and empty. By default, empty means that Carbondata will not compress the sort temp files. This parameter will be useful if you encounter disk bottleneck. |
+ | carbon.load.skewedDataOptimization.enabled | spark/carbonlib/carbon.properties | Data loading | Whether to enable size based block allocation strategy for data loading. | When loading, carbondata will use file size based block allocation strategy for task distribution. It will make sure that all the executors process the same size of data -- It's useful if the size of your input data files varies widely, say 1MB~1GB. |
Note: If your CarbonData instance is provided only for query, you may specify the property 'spark.speculation=true' which is in conf directory of spark.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 06acbba..8ba2767 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -281,8 +281,10 @@ class NewCarbonDataLoadRDD[K, V](
val format = new CSVInputFormat
val split = theSplit.asInstanceOf[CarbonNodePartition]
+ val inputSize = split.blocksDetails.map(_.getBlockLength).sum * 0.1 * 10 / 1024 / 1024
logInfo("Input split: " + split.serializableHadoopSplit)
- logInfo("The Block Count in this node :" + split.nodeBlocksDetail.length)
+ logInfo("The block count in this node: " + split.nodeBlocksDetail.length)
+ logInfo(f"The input data size in this node: $inputSize%.2fMB")
CarbonTimeStatisticsFactory.getLoadStatisticsInstance.recordHostBlockMap(
split.serializableHadoopSplit, split.nodeBlocksDetail.length)
carbonLoadModel.setTaskNo(String.valueOf(theSplit.index))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
index 1958d61..a676dd8 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
@@ -128,7 +128,7 @@ object DistributionUtil {
*/
def ensureExecutorsAndGetNodeList(blockList: Seq[Distributable],
sparkContext: SparkContext): Seq[String] = {
- val nodeMapping = CarbonLoaderUtil.getRequiredExecutors(blockList.asJava)
+ val nodeMapping = CarbonLoaderUtil.nodeBlockMapping(blockList.asJava)
ensureExecutorsByNumberAndGetNodeList(nodeMapping, blockList, sparkContext)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 2c4d604..25ddd0b 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -1029,10 +1029,16 @@ object CarbonDataRDDFactory {
val startTime = System.currentTimeMillis
val activeNodes = DistributionUtil
.ensureExecutorsAndGetNodeList(blockList, sqlContext.sparkContext)
- val nodeBlockMapping =
- CarbonLoaderUtil
- .nodeBlockMapping(blockList.toSeq.asJava, -1, activeNodes.toList.asJava).asScala
- .toSeq
+ val skewedDataOptimization = CarbonProperties.getInstance()
+ .isLoadSkewedDataOptimizationEnabled()
+ val blockAssignStrategy = if (skewedDataOptimization) {
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_SIZE_FIRST
+ } else {
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST
+ }
+ LOGGER.info(s"Allocating block to nodes using strategy: $blockAssignStrategy")
+ val nodeBlockMapping = CarbonLoaderUtil.nodeBlockMapping(blockList.toSeq.asJava, -1,
+ activeNodes.toList.asJava, blockAssignStrategy).asScala.toSeq
val timeElapsed: Long = System.currentTimeMillis - startTime
LOGGER.info("Total Time taken in block allocation: " + timeElapsed)
LOGGER.info(s"Total no of blocks: ${ blockList.length }, " +
@@ -1040,7 +1046,9 @@ object CarbonDataRDDFactory {
var str = ""
nodeBlockMapping.foreach { entry =>
val tableBlock = entry._2
- str = str + "#Node: " + entry._1 + " no.of.blocks: " + tableBlock.size()
+ val totalSize = tableBlock.asScala.map(_.asInstanceOf[TableBlockInfo].getBlockLength).sum
+ str = str + "#Node: " + entry._1 + ", no.of.blocks: " + tableBlock.size() +
+ f", totalsize.of.blocks: ${totalSize * 0.1 * 10 / 1024 /1024}%.2fMB"
tableBlock.asScala.foreach(tableBlockInfo =>
if (!tableBlockInfo.getLocations.exists(hostentry =>
hostentry.equalsIgnoreCase(entry._1)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/processing/src/main/java/org/apache/carbondata/processing/merger/NodeMultiBlockRelation.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/NodeMultiBlockRelation.java b/processing/src/main/java/org/apache/carbondata/processing/merger/NodeMultiBlockRelation.java
index ec2ddaf..1bb5736 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/NodeMultiBlockRelation.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/NodeMultiBlockRelation.java
@@ -16,15 +16,41 @@
*/
package org.apache.carbondata.processing.merger;
+import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.core.datastore.block.Distributable;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
public class NodeMultiBlockRelation implements Comparable<NodeMultiBlockRelation> {
private final List<Distributable> blocks;
private final String node;
+ /**
+ * comparator to sort by data size in descending order. This is used to assign big blocks to
+ * bigger nodes first.
+ */
+ public static final Comparator<NodeMultiBlockRelation> DATA_SIZE_DESC_COMPARATOR =
+ new Comparator<NodeMultiBlockRelation>() {
+ @Override
+ public int compare(NodeMultiBlockRelation o1, NodeMultiBlockRelation o2) {
+ long diff = o1.getTotalSizeOfBlocks() - o2.getTotalSizeOfBlocks();
+ return diff > 0 ? -1 : (diff < 0 ? 1 : 0);
+ }
+ };
+ /**
+ * comparator to sort by data size in ascending order. This is used to assign left over blocks to
+ * smaller nodes first.
+ */
+ public static final Comparator<NodeMultiBlockRelation> DATA_SIZE_ASC_COMPARATOR =
+ new Comparator<NodeMultiBlockRelation>() {
+ @Override
+ public int compare(NodeMultiBlockRelation o1, NodeMultiBlockRelation o2) {
+ long diff = o1.getTotalSizeOfBlocks() - o2.getTotalSizeOfBlocks();
+ return diff > 0 ? 1 : (diff < 0 ? -1 : 0);
+ }
+ };
public NodeMultiBlockRelation(String node, List<Distributable> blocks) {
this.node = node;
this.blocks = blocks;
@@ -39,6 +65,20 @@ public class NodeMultiBlockRelation implements Comparable<NodeMultiBlockRelation
return node;
}
+ /**
+ * get the total size of the blocks
+ * @return size in bytes
+ */
+ public long getTotalSizeOfBlocks() {
+ long totalSize = 0;
+ if (blocks.get(0) instanceof TableBlockInfo) {
+ for (Distributable block : blocks) {
+ totalSize += ((TableBlockInfo) block).getBlockLength();
+ }
+ }
+ return totalSize;
+ }
+
@Override public int compareTo(NodeMultiBlockRelation obj) {
return this.blocks.size() - obj.getBlocks().size();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 5df39bc..38e5698 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentif
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.Distributable;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -50,7 +51,6 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.processing.merger.NodeBlockRelation;
import org.apache.carbondata.processing.merger.NodeMultiBlockRelation;
import static org.apache.carbondata.core.enums.EscapeSequences.*;
@@ -64,6 +64,23 @@ public final class CarbonLoaderUtil {
private CarbonLoaderUtil() {
}
+ /**
+ * strategy for assign blocks to nodes/executors
+ */
+ public enum BlockAssignmentStrategy {
+ BLOCK_NUM_FIRST("Assign blocks to node base on number of blocks"),
+ BLOCK_SIZE_FIRST("Assign blocks to node base on data size of blocks");
+ private String name;
+ BlockAssignmentStrategy(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + ':' + this.name;
+ }
+ }
+
public static void deleteSegment(CarbonLoadModel loadModel, int currentLoad) {
String segmentPath = CarbonTablePath.getSegmentPath(
loadModel.getTablePath(), currentLoad + "");
@@ -530,9 +547,9 @@ public final class CarbonLoaderUtil {
public static Map<String, List<List<Distributable>>> nodeBlockTaskMapping(
List<Distributable> blockInfos, int noOfNodesInput, int parallelism,
List<String> activeNode) {
-
Map<String, List<Distributable>> mapOfNodes =
- CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode);
+ CarbonLoaderUtil.nodeBlockMapping(blockInfos, noOfNodesInput, activeNode,
+ BlockAssignmentStrategy.BLOCK_NUM_FIRST);
int taskPerNode = parallelism / mapOfNodes.size();
//assigning non zero value to noOfTasksPerNode
int noOfTasksPerNode = taskPerNode == 0 ? 1 : taskPerNode;
@@ -548,86 +565,75 @@ public final class CarbonLoaderUtil {
*/
public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos,
int noOfNodesInput) {
- return nodeBlockMapping(blockInfos, noOfNodesInput, null);
+ return nodeBlockMapping(blockInfos, noOfNodesInput, null,
+ BlockAssignmentStrategy.BLOCK_NUM_FIRST);
}
/**
- * the method returns the number of required executors
+ * This method will divide the blocks among the nodes as per the data locality
*
* @param blockInfos
* @return
*/
- public static Map<String, List<Distributable>> getRequiredExecutors(
- List<Distributable> blockInfos) {
- List<NodeBlockRelation> flattenedList =
- new ArrayList<NodeBlockRelation>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- for (Distributable blockInfo : blockInfos) {
- try {
- for (String eachNode : blockInfo.getLocations()) {
- NodeBlockRelation nbr = new NodeBlockRelation(blockInfo, eachNode);
- flattenedList.add(nbr);
- }
- } catch (IOException e) {
- throw new RuntimeException("error getting location of block: " + blockInfo.toString(), e);
- }
- }
- // sort the flattened data.
- Collections.sort(flattenedList);
- Map<String, List<Distributable>> nodeAndBlockMapping =
- new LinkedHashMap<String, List<Distributable>>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // from the flattened list create a mapping of node vs Data blocks.
- createNodeVsBlockMapping(flattenedList, nodeAndBlockMapping);
- return nodeAndBlockMapping;
+ public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos) {
+ // -1 if number of nodes has to be decided based on block location information
+ return nodeBlockMapping(blockInfos, -1);
}
/**
* This method will divide the blocks among the nodes as per the data locality
*
- * @param blockInfos
+ * @param blockInfos blocks
* @param noOfNodesInput -1 if number of nodes has to be decided
* based on block location information
- * @return
+ * @param blockAssignmentStrategy strategy used to assign blocks
+ * @return a map that maps node to blocks
*/
- public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos,
- int noOfNodesInput, List<String> activeNodes) {
-
- Map<String, List<Distributable>> nodeBlocksMap =
- new HashMap<String, List<Distributable>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- List<NodeBlockRelation> flattenedList =
- new ArrayList<NodeBlockRelation>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- Set<Distributable> uniqueBlocks =
- new HashSet<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- Set<String> nodes = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- createFlattenedListFromMap(blockInfos, flattenedList, uniqueBlocks, nodes);
+ public static Map<String, List<Distributable>> nodeBlockMapping(
+ List<Distributable> blockInfos, int noOfNodesInput, List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ ArrayList<NodeMultiBlockRelation> rtnNode2Blocks = new ArrayList<>();
+
+ Set<Distributable> uniqueBlocks = new HashSet<>(blockInfos);
+ ArrayList<NodeMultiBlockRelation> originNode2Blocks = createNode2BlocksMapping(blockInfos);
+ Set<String> nodes = new HashSet<>(originNode2Blocks.size());
+ for (NodeMultiBlockRelation relation : originNode2Blocks) {
+ nodes.add(relation.getNode());
+ }
int noofNodes = (-1 == noOfNodesInput) ? nodes.size() : noOfNodesInput;
if (null != activeNodes) {
noofNodes = activeNodes.size();
}
- int blocksPerNode = blockInfos.size() / noofNodes;
- blocksPerNode = blocksPerNode <= 0 ? 1 : blocksPerNode;
-
- // sort the flattened data.
- Collections.sort(flattenedList);
- Map<String, List<Distributable>> nodeAndBlockMapping =
- new LinkedHashMap<String, List<Distributable>>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- // from the flattened list create a mapping of node vs Data blocks.
- createNodeVsBlockMapping(flattenedList, nodeAndBlockMapping);
+ // calculate the average expected size for each node
+ long sizePerNode = 0;
+ if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
+ sizePerNode = blockInfos.size() / noofNodes;
+ sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
+ } else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy) {
+ long totalFileSize = 0;
+ for (Distributable blockInfo : uniqueBlocks) {
+ totalFileSize += ((TableBlockInfo) blockInfo).getBlockLength();
+ }
+ sizePerNode = totalFileSize / noofNodes;
+ }
- // so now we have a map of node vs blocks. allocate the block as per the order
- createOutputMap(nodeBlocksMap, blocksPerNode, uniqueBlocks, nodeAndBlockMapping, activeNodes);
+ // assign blocks to each node
+ assignBlocksByDataLocality(rtnNode2Blocks, sizePerNode, uniqueBlocks, originNode2Blocks,
+ activeNodes, blockAssignmentStrategy);
// if any blocks remain then assign them to nodes in round robin.
- assignLeftOverBlocks(nodeBlocksMap, uniqueBlocks, blocksPerNode, activeNodes);
+ assignLeftOverBlocks(rtnNode2Blocks, uniqueBlocks, sizePerNode, activeNodes,
+ blockAssignmentStrategy);
- return nodeBlocksMap;
+ // convert
+ Map<String, List<Distributable>> rtnNodeBlocksMap =
+ new HashMap<String, List<Distributable>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ for (NodeMultiBlockRelation relation : rtnNode2Blocks) {
+ rtnNodeBlocksMap.put(relation.getNode(), relation.getBlocks());
+ }
+ return rtnNodeBlocksMap;
}
/**
@@ -702,92 +708,207 @@ public final class CarbonLoaderUtil {
}
/**
- * If any left over data blocks are present then assign those to nodes in round robin way.
- *
- * @param outputMap
- * @param uniqueBlocks
+ * If any left over data blocks are present then assign those to nodes in round robin way. This
+ * will not obey the data locality.
*/
- private static void assignLeftOverBlocks(Map<String, List<Distributable>> outputMap,
- Set<Distributable> uniqueBlocks, int noOfBlocksPerNode, List<String> activeNodes) {
+ private static void assignLeftOverBlocks(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> leftOverBlocks, long expectedSizePerNode, List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ Map<String, Integer> node2Idx = new HashMap<>(outputMap.size());
+ for (int idx = 0; idx < outputMap.size(); idx++) {
+ node2Idx.put(outputMap.get(idx).getNode(), idx);
+ }
+ // iterate all the nodes and try to allocate blocks to the nodes
if (activeNodes != null) {
for (String activeNode : activeNodes) {
- List<Distributable> blockLst = outputMap.get(activeNode);
- if (null == blockLst) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second assignment iteration: assign for executor: " + activeNode);
+ }
+
+ Integer idx;
+ List<Distributable> blockLst;
+ if (node2Idx.containsKey(activeNode)) {
+ idx = node2Idx.get(activeNode);
+ blockLst = outputMap.get(idx).getBlocks();
+ } else {
+ idx = node2Idx.size();
blockLst = new ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
- if (blockLst.size() > 0) {
- outputMap.put(activeNode, blockLst);
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst, blockAssignmentStrategy);
+
+ if (!node2Idx.containsKey(activeNode) && blockLst.size() > 0) {
+ outputMap.add(idx, new NodeMultiBlockRelation(activeNode, blockLst));
+ node2Idx.put(activeNode, idx);
}
}
} else {
- for (Map.Entry<String, List<Distributable>> entry : outputMap.entrySet()) {
- List<Distributable> blockLst = entry.getValue();
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
+ for (NodeMultiBlockRelation entry : outputMap) {
+ List<Distributable> blockLst = entry.getBlocks();
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst, blockAssignmentStrategy);
}
-
}
- for (Map.Entry<String, List<Distributable>> entry : outputMap.entrySet()) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- if (blocks.hasNext()) {
- Distributable block = blocks.next();
- List<Distributable> blockLst = entry.getValue();
- blockLst.add(block);
- blocks.remove();
- }
+ // if there is still blocks left, allocate them in round robin manner to each nodes
+ assignBlocksUseRoundRobin(outputMap, leftOverBlocks, blockAssignmentStrategy);
+ }
+
+ /**
+ * assign remaining blocks to nodes
+ *
+ * @param remainingBlocks blocks to be allocated
+ * @param expectedSizePerNode expected size for each node
+ * @param blockLst destination for the blocks to be allocated
+ * @param blockAssignmentStrategy block assignment stretegy
+ */
+ private static void populateBlocks(Set<Distributable> remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ switch (blockAssignmentStrategy) {
+ case BLOCK_NUM_FIRST:
+ populateBlocksByNum(remainingBlocks, expectedSizePerNode, blockLst);
+ break;
+ case BLOCK_SIZE_FIRST:
+ populateBlocksBySize(remainingBlocks, expectedSizePerNode, blockLst);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported block assignment strategy: " + blockAssignmentStrategy);
}
}
/**
- * The method populate the blockLst to be allocate to a specific node.
- * @param uniqueBlocks
- * @param noOfBlocksPerNode
- * @param blockLst
+ * Taken N number of distributable blocks from {@param remainingBlocks} and add them to output
+ * {@param blockLst}. After added, the total number of {@param blockLst} is less
+ * than {@param expectedSizePerNode}.
*/
- private static void populateBlocks(Set<Distributable> uniqueBlocks, int noOfBlocksPerNode,
- List<Distributable> blockLst) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- //if the node is already having the per block nodes then avoid assign the extra blocks
- if (blockLst.size() == noOfBlocksPerNode) {
+ private static void populateBlocksByNum(Set<Distributable> remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ // if the node is already having the per block nodes then avoid assign the extra blocks
+ if (blockLst.size() == expectedSizePerNode) {
return;
}
while (blocks.hasNext()) {
Distributable block = blocks.next();
blockLst.add(block);
blocks.remove();
- if (blockLst.size() >= noOfBlocksPerNode) {
+ if (blockLst.size() >= expectedSizePerNode) {
break;
}
}
}
/**
- * To create the final output of the Node and Data blocks
- *
- * @param outputMap
- * @param blocksPerNode
- * @param uniqueBlocks
- * @param nodeAndBlockMapping
- * @param activeNodes
+ * Taken N number of distributable blocks from {@param remainingBlocks} and add them to output
+ * {@param blockLst}. After added, the total accumulated block size of {@param blockLst}
+ * is less than {@param expectedSizePerNode}.
*/
- private static void createOutputMap(Map<String, List<Distributable>> outputMap, int blocksPerNode,
- Set<Distributable> uniqueBlocks, Map<String, List<Distributable>> nodeAndBlockMapping,
- List<String> activeNodes) {
+ private static void populateBlocksBySize(Set<Distributable> remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ //if the node is already having the avg node size then avoid assign the extra blocks
+ long fileSize = 0;
+ for (Distributable block : blockLst) {
+ fileSize += ((TableBlockInfo) block).getBlockLength();
+ }
+ if (fileSize >= expectedSizePerNode) {
+ LOGGER.debug("Capacity is full, skip allocate blocks on this node");
+ return;
+ }
- ArrayList<NodeMultiBlockRelation> multiBlockRelations =
- new ArrayList<>(nodeAndBlockMapping.size());
- for (Map.Entry<String, List<Distributable>> entry : nodeAndBlockMapping.entrySet()) {
- multiBlockRelations.add(new NodeMultiBlockRelation(entry.getKey(), entry.getValue()));
+ while (blocks.hasNext()) {
+ Distributable block = blocks.next();
+ long thisBlockSize = ((TableBlockInfo) block).getBlockLength();
+ if (fileSize < expectedSizePerNode) {
+ // `fileSize==0` means there are no blocks assigned to this node before
+ if (fileSize == 0 || fileSize + thisBlockSize <= expectedSizePerNode * 1.1D) {
+ blockLst.add(block);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second Assignment iteration: "
+ + ((TableBlockInfo) block).getFilePath() + "-"
+ + ((TableBlockInfo) block).getBlockLength() + "-->currentNode");
+ }
+ fileSize += thisBlockSize;
+ blocks.remove();
+ }
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * allocate the blocks in round robin manner
+ */
+ private static void assignBlocksUseRoundRobin(ArrayList<NodeMultiBlockRelation> node2Blocks,
+ Set<Distributable> remainingBlocks, BlockAssignmentStrategy blockAssignmentStrategy) {
+ switch (blockAssignmentStrategy) {
+ case BLOCK_NUM_FIRST:
+ roundRobinAssignBlocksByNum(node2Blocks, remainingBlocks);
+ break;
+ case BLOCK_SIZE_FIRST:
+ roundRobinAssignBlocksBySize(node2Blocks, remainingBlocks);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported block assignment strategy: "
+ + blockAssignmentStrategy);
+ }
+ }
+
+ private static void roundRobinAssignBlocksByNum(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> remainingBlocks) {
+ for (NodeMultiBlockRelation relation: outputMap) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ if (blocks.hasNext()) {
+ Distributable block = blocks.next();
+ List<Distributable> blockLst = relation.getBlocks();
+ blockLst.add(block);
+ blocks.remove();
+ }
+ }
+ }
+
+ private static void roundRobinAssignBlocksBySize(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> remainingBlocks) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ while (blocks.hasNext()) {
+ // sort the allocated node-2-blocks in ascending order, the total data size of first one is
+ // the smallest, so we assign this block to it.
+ Collections.sort(outputMap, NodeMultiBlockRelation.DATA_SIZE_ASC_COMPARATOR);
+ Distributable block = blocks.next();
+ List<Distributable> blockLst = outputMap.get(0).getBlocks();
+ blockLst.add(block);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("RoundRobin assignment iteration: "
+ + ((TableBlockInfo) block).getFilePath() + "-"
+ + ((TableBlockInfo) block).getBlockLength() + "-->" + outputMap.get(0).getNode());
+ }
+ blocks.remove();
+ }
+ }
+ /**
+ * allocate distributable blocks to nodes based on data locality
+ */
+ private static void assignBlocksByDataLocality(
+ ArrayList<NodeMultiBlockRelation> outputNode2Blocks,
+ long expectedSizePerNode, Set<Distributable> remainingBlocks,
+ List<NodeMultiBlockRelation> inputNode2Blocks, List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy) {
+ // sort nodes based on data size of all blocks per node, so that nodes having bigger size
+ // are assigned first
+ Collections.sort(inputNode2Blocks, NodeMultiBlockRelation.DATA_SIZE_DESC_COMPARATOR);
+ } else {
+ // sort nodes based on number of blocks per node, so that nodes having lesser blocks
+ // are assigned first
+ Collections.sort(inputNode2Blocks);
}
- // sort nodes based on number of blocks per node, so that nodes having lesser blocks
- // are assigned first
- Collections.sort(multiBlockRelations);
- for (NodeMultiBlockRelation nodeMultiBlockRelation : multiBlockRelations) {
+ Map<String, Integer> executor2Idx = new HashMap<>();
+ for (NodeMultiBlockRelation nodeMultiBlockRelation : inputNode2Blocks) {
String nodeName = nodeMultiBlockRelation.getNode();
- //assign the block to the node only if the node is active
+ // assign the block to the node only if the node is active
String activeExecutor = nodeName;
if (null != activeNodes) {
activeExecutor = getActiveExecutor(activeNodes, nodeName);
@@ -795,29 +916,75 @@ public final class CarbonLoaderUtil {
continue;
}
}
- // this loop will be for each NODE
- int nodeCapacity = 0;
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("First Assignment iteration: assign for executor: " + activeExecutor);
+ }
+
+ List<Distributable> blocksInThisNode = nodeMultiBlockRelation.getBlocks();
+ if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy) {
+ // sort blocks based on block size, so that bigger blocks will be assigned first
+ Collections.sort(blocksInThisNode, TableBlockInfo.DATA_SIZE_DESC_COMPARATOR);
+ }
+
+ long nodeCapacity = 0;
// loop thru blocks of each Node
for (Distributable block : nodeMultiBlockRelation.getBlocks()) {
+ if (!remainingBlocks.contains(block)) {
+ // this block has been added before
+ continue;
+ }
+ // this is the first time to add block to this node, initialize it
+ if (!executor2Idx.containsKey(activeExecutor)) {
+ Integer idx = executor2Idx.size();
+ outputNode2Blocks.add(idx, new NodeMultiBlockRelation(activeExecutor,
+ new ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE)));
+ executor2Idx.put(activeExecutor, idx);
+ }
- // check if this is already assigned.
- if (uniqueBlocks.contains(block)) {
-
- if (null == outputMap.get(activeExecutor)) {
- List<Distributable> list =
- new ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- outputMap.put(activeExecutor, list);
- }
- // assign this block to this node if node has capacity left
- if (nodeCapacity < blocksPerNode) {
- List<Distributable> infos = outputMap.get(activeExecutor);
+ // assign this block to this node if node has capacity left
+ if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
+ if (nodeCapacity < expectedSizePerNode) {
+ Integer idx = executor2Idx.get(activeExecutor);
+ List<Distributable> infos = outputNode2Blocks.get(idx).getBlocks();
infos.add(block);
nodeCapacity++;
- uniqueBlocks.remove(block);
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "First Assignment iteration: " + ((TableBlockInfo) block).getFilePath() + '-'
+ + ((TableBlockInfo) block).getBlockLength() + "-->" + activeExecutor);
+ }
+ remainingBlocks.remove(block);
+ } else {
+ // No need to continue loop as node is full
+ break;
+ }
+ } else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == blockAssignmentStrategy) {
+ long thisBlockSize = ((TableBlockInfo) block).getBlockLength();
+ // `nodeCapacity == 0` means that there is a huge block that already exceed the
+ // `expectedSize` of the node, so we have to assign it to some node, otherwise it will
+ // be assigned in the last RoundRobin iteration.
+ if (nodeCapacity == 0 || nodeCapacity < expectedSizePerNode) {
+ if (nodeCapacity == 0 || nodeCapacity + thisBlockSize <= expectedSizePerNode * 1.05D) {
+ Integer idx = executor2Idx.get(activeExecutor);
+ List<Distributable> blocks = outputNode2Blocks.get(idx).getBlocks();
+ blocks.add(block);
+ nodeCapacity += thisBlockSize;
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "First Assignment iteration: " + ((TableBlockInfo) block).getFilePath() + '-'
+ + ((TableBlockInfo) block).getBlockLength() + "-->" + activeExecutor);
+ }
+ remainingBlocks.remove(block);
+ }
+ // this block is too big for current node and there are still capacity left
+ // for small files, so continue to allocate block on this node in next iteration.
} else {
// No need to continue loop as node is full
break;
}
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported block assignment strategy: " + blockAssignmentStrategy);
}
}
}
@@ -861,60 +1028,37 @@ public final class CarbonLoaderUtil {
}
/**
- * Create the Node and its related blocks Mapping and put in a Map
+ * Create node to blocks mapping
*
- * @param flattenedList
- * @param nodeAndBlockMapping
+ * @param blockInfos input block info
*/
- private static void createNodeVsBlockMapping(List<NodeBlockRelation> flattenedList,
- Map<String, List<Distributable>> nodeAndBlockMapping) {
- for (NodeBlockRelation nbr : flattenedList) {
- String node = nbr.getNode();
- List<Distributable> list;
-
- if (null == nodeAndBlockMapping.get(node)) {
- list = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- list.add(nbr.getBlock());
- nodeAndBlockMapping.put(node, list);
- } else {
- list = nodeAndBlockMapping.get(node);
- list.add(nbr.getBlock());
- }
- }
- /*for resolving performance issue, removed values() with entrySet () iterating the values and
- sorting it.entrySet will give the logical view for hashMap and we dont query the map twice for
- each key whereas values () iterate twice*/
- Iterator<Map.Entry<String, List<Distributable>>> iterator =
- nodeAndBlockMapping.entrySet().iterator();
- while (iterator.hasNext()) {
- Collections.sort(iterator.next().getValue());
- }
- }
+ private static ArrayList<NodeMultiBlockRelation> createNode2BlocksMapping(
+ List<Distributable> blockInfos) {
+ Map<String, Integer> node2Idx = new HashMap<>();
+ ArrayList<NodeMultiBlockRelation> node2Blocks = new ArrayList<>();
- /**
- * Create the flat List i.e flattening of the Map.
- *
- * @param blockInfos
- * @param flattenedList
- * @param uniqueBlocks
- */
- private static void createFlattenedListFromMap(List<Distributable> blockInfos,
- List<NodeBlockRelation> flattenedList, Set<Distributable> uniqueBlocks,
- Set<String> nodeList) {
for (Distributable blockInfo : blockInfos) {
- // put the blocks in the set
- uniqueBlocks.add(blockInfo);
-
try {
- for (String eachNode : blockInfo.getLocations()) {
- NodeBlockRelation nbr = new NodeBlockRelation(blockInfo, eachNode);
- flattenedList.add(nbr);
- nodeList.add(eachNode);
+ for (final String eachNode : blockInfo.getLocations()) {
+ if (node2Idx.containsKey(eachNode)) {
+ Integer idx = node2Idx.get(eachNode);
+ List<Distributable> blocks = node2Blocks.get(idx).getBlocks();
+ blocks.add(blockInfo);
+ } else {
+ // add blocks to this node for the first time
+ Integer idx = node2Idx.size();
+ List<Distributable> blocks = new ArrayList<>();
+ blocks.add(blockInfo);
+ node2Blocks.add(idx, new NodeMultiBlockRelation(eachNode, blocks));
+ node2Idx.put(eachNode, idx);
+ }
}
} catch (IOException e) {
throw new RuntimeException("error getting location of block: " + blockInfo.toString(), e);
}
}
+
+ return node2Blocks;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9a423c24/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java b/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
new file mode 100644
index 0000000..9c66ada
--- /dev/null
+++ b/processing/src/test/java/org/apache/carbondata/processing/util/CarbonLoaderUtilTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.block.Distributable;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CarbonLoaderUtilTest {
+ private final static LogService LOGGER
+ = LogServiceFactory.getLogService(CarbonLoaderUtilTest.class.getName());
+
+ private List<Distributable> generateBlocks() {
+ List<Distributable> blockInfos = new ArrayList<>();
+ String filePath = "/fakepath";
+ String blockId = "1";
+
+ String[] locations = new String[] { "host2", "host3" };
+ ColumnarFormatVersion version = ColumnarFormatVersion.V1;
+
+ TableBlockInfo tableBlockInfo1 = new TableBlockInfo(filePath + "_a", 0,
+ blockId, locations, 30 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo1);
+
+ TableBlockInfo tableBlockInfo2 = new TableBlockInfo(filePath + "_b", 0,
+ blockId, locations, 40 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo2);
+
+ TableBlockInfo tableBlockInfo3 = new TableBlockInfo(filePath + "_c", 0,
+ blockId, locations, 20 * 1024 * 1024, version, null);
+ blockInfos.add(tableBlockInfo3);
+
+ TableBlockInfo tableBlockInfo4 = new TableBlockInfo(filePath + "_d", 0,
+ blockId, locations, 1, version, null);
+ blockInfos.add(tableBlockInfo4);
+
+ TableBlockInfo tableBlockInfo5 = new TableBlockInfo(filePath + "_e", 0,
+ blockId, locations, 1, version, null);
+ blockInfos.add(tableBlockInfo5);
+
+ TableBlockInfo tableBlockInfo6 = new TableBlockInfo(filePath + "_f", 0,
+ blockId, locations, 1, version, null);
+ blockInfos.add(tableBlockInfo6);
+
+ TableBlockInfo tableBlockInfo7 = new TableBlockInfo(filePath + "_g", 0,
+ blockId, locations, 1, version, null);
+ blockInfos.add(tableBlockInfo7);
+ return blockInfos;
+ }
+
+ private List<String> generateExecutors() {
+ List<String> activeNodes = new ArrayList<>();
+ activeNodes.add("host1");
+ activeNodes.add("host2");
+ activeNodes.add("host3");
+ return activeNodes;
+ }
+
+ @Test
+ public void testNodeBlockMappingByDataSize() throws Exception {
+ List<Distributable> blockInfos = generateBlocks();
+ List<String> activeNodes = generateExecutors();
+
+ // the blocks are assigned by size, so the number of block for each node are different
+ Map<String, List<Distributable>> nodeMappingBySize =
+ CarbonLoaderUtil.nodeBlockMapping(blockInfos, -1, activeNodes,
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_SIZE_FIRST);
+ LOGGER.info(convertMapListAsString(nodeMappingBySize));
+ Assert.assertEquals(3, nodeMappingBySize.size());
+ for (Map.Entry<String, List<Distributable>> entry : nodeMappingBySize.entrySet()) {
+ if (entry.getValue().size() == 1) {
+ // only contains the biggest block
+ Assert.assertEquals(40 * 1024 * 1024L,
+ ((TableBlockInfo) entry.getValue().get(0)).getBlockLength());
+ } else {
+ Assert.assertTrue(entry.getValue().size() > 1);
+ }
+ }
+
+ // the blocks are assigned by number, so the number of blocks for each node are nearly the same
+ Map<String, List<Distributable>> nodeMappingByNum =
+ CarbonLoaderUtil.nodeBlockMapping(blockInfos, -1, activeNodes,
+ CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST);
+ LOGGER.info(convertMapListAsString(nodeMappingByNum));
+ Assert.assertEquals(3, nodeMappingBySize.size());
+ for (Map.Entry<String, List<Distributable>> entry : nodeMappingByNum.entrySet()) {
+ Assert.assertTrue(entry.getValue().size() == blockInfos.size() / 3
+ || entry.getValue().size() == blockInfos.size() / 3 + 1);
+ }
+ }
+
+ private <K, T> String convertMapListAsString(Map<K, List<T>> mapList) {
+ StringBuffer sb = new StringBuffer();
+ for (Map.Entry<K, List<T>> entry : mapList.entrySet()) {
+ String key = entry.getKey().toString();
+ String value = StringUtils.join(entry.getValue(), ", ");
+ sb.append(key).append(" -- ").append(value).append(System.lineSeparator());
+ }
+ return sb.toString();
+ }
+}
\ No newline at end of file
[54/54] [abbrv] carbondata git commit: [HOTFIX] Fixed SDV Tests
Posted by ra...@apache.org.
[HOTFIX] Fixed SDV Tests
This closes #2030
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/39fa1eb5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/39fa1eb5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/39fa1eb5
Branch: refs/heads/master
Commit: 39fa1eb5830d6fc4ee4f6abd984b03ec7d426192
Parents: d23f7fa
Author: ravipesala <ra...@gmail.com>
Authored: Thu Mar 8 10:21:31 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:24:18 2018 +0530
----------------------------------------------------------------------
.../carbondata/cluster/sdv/generated/MergeIndexTestCase.scala | 3 ++-
.../apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala | 4 +++-
.../scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala | 3 ++-
3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39fa1eb5/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index b886b11..ff421e1 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -84,10 +84,11 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/VmaLL100' INTO TABLE carbon_automation_nonmerge OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSy
sVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription')""").collect
val rows = sql("""Select count(*) from carbon_automation_nonmerge""").collect()
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0") == 2)
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 2)
- assert(getIndexFileCount("default", "carbon_automation_nonmerge", "1") == 2)
+ assert(getIndexFileCount("default", "carbon_automation_nonmerge", "2") == 2)
sql("ALTER TABLE carbon_automation_nonmerge COMPACT 'minor'").collect()
val table = CarbonMetadata.getInstance().getCarbonTable("default","carbon_automation_nonmerge")
new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39fa1eb5/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
index 9de8dc9..85a6f41 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableLoadPartitionRDD.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.execution.command.AlterPartitionModel
import org.apache.spark.util.PartitionUtils
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.partition.spliter.RowResultProcessor
@@ -64,6 +64,8 @@ class AlterTableLoadPartitionRDD[K, V](alterPartitionModel: AlterPartitionModel,
val partitionId = partitionInfo.getPartitionId(split.index)
carbonLoadModel.setTaskNo(String.valueOf(partitionId))
carbonLoadModel.setSegmentId(segmentId)
+ CarbonMetadata.getInstance().addCarbonTable(
+ carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable)
CommonUtil.setTempStoreLocation(split.index, carbonLoadModel, false, true)
val tempLocationKey = CarbonDataProcessorUtil
.getTempStoreLocationKey(carbonLoadModel.getDatabaseName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/39fa1eb5/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 444c9dc..da268c1 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -39,7 +39,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block._
import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.mutate.UpdateVO
@@ -83,6 +83,7 @@ class CarbonMergerRDD[K, V](
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val iter = new Iterator[(K, V)] {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
+ CarbonMetadata.getInstance().addCarbonTable(carbonTable)
val carbonSparkPartition = theSplit.asInstanceOf[CarbonSparkPartition]
if (carbonTable.isPartitionTable) {
carbonLoadModel.setTaskNo(String.valueOf(carbonSparkPartition.partitionId))
[44/54] [abbrv] carbondata git commit: [CARBONDATA-2206] support
lucene index datamap
Posted by ra...@apache.org.
[CARBONDATA-2206] support lucene index datamap
This PR is an initial effort to integrate lucene as an index datamap into carbondata.
A new module called carbondata-lucene is added to support lucene datamap:
1.Add LuceneFineGrainDataMap, implement FineGrainDataMap interface.
2.Add LuceneCoarseGrainDataMap, implement CoarseGrainDataMap interface.
3.Support writing lucene index via LuceneDataMapWriter.
4.Implement LuceneDataMapFactory
5.A UDF called TEXT_MATCH is added, use it to do filtering on string column by lucene
This closes #2003
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bbb10922
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bbb10922
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bbb10922
Branch: refs/heads/master
Commit: bbb10922b3ed7ddf12881dae339848f1d88b0984
Parents: a80db8e
Author: Jacky Li <ja...@qq.com>
Authored: Mon Feb 26 16:30:38 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapChooser.java | 4 +
.../core/datamap/DataMapStoreManager.java | 5 +-
.../carbondata/core/datamap/dev/DataMap.java | 2 +-
.../core/datamap/dev/DataMapFactory.java | 2 +-
.../core/datamap/dev/DataMapWriter.java | 7 +-
.../cgdatamap/CoarseGrainDataMapFactory.java | 1 +
.../core/scan/filter/intf/ExpressionType.java | 3 +-
datamap/lucene/pom.xml | 149 +++++++++
.../lucene/LuceneCoarseGrainDataMap.java | 232 +++++++++++++
.../lucene/LuceneCoarseGrainDataMapFactory.java | 72 ++++
.../lucene/LuceneDataMapDistributable.java | 36 ++
.../lucene/LuceneDataMapFactoryBase.java | 180 ++++++++++
.../datamap/lucene/LuceneDataMapWriter.java | 328 +++++++++++++++++++
.../datamap/lucene/LuceneFineGrainDataMap.java | 280 ++++++++++++++++
.../lucene/LuceneFineGrainDataMapFactory.java | 68 ++++
.../lucene/LuceneCoarseGrainDataMapSuite.scala | 73 +++++
.../lucene/LuceneFineGrainDataMapSuite.scala | 98 ++++++
integration/spark-common-test/pom.xml | 6 +
.../testsuite/datamap/FGDataMapTestCase.scala | 2 +-
.../carbondata/datamap/DataMapProvider.java | 4 +-
.../datamap/IndexDataMapProvider.java | 4 +-
.../datamap/expression/MatchExpression.java | 56 ++++
.../carbondata/datamap/TextMatchUDF.scala | 34 ++
.../scala/org/apache/spark/sql/CarbonEnv.scala | 5 +
.../strategy/CarbonLateDecodeStrategy.scala | 9 +
.../spark/sql/optimizer/CarbonFilters.scala | 4 +
pom.xml | 3 +
.../datamap/DataMapWriterListener.java | 6 +-
.../store/writer/AbstractFactDataWriter.java | 12 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 4 +-
30 files changed, 1671 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index 94b48c6..c8c971d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -228,6 +228,10 @@ public class DataMapChooser {
private boolean contains(DataMapMeta mapMeta, List<ColumnExpression> columnExpressions,
Set<ExpressionType> expressionTypes) {
+ if (mapMeta.getOptimizedOperation().contains(ExpressionType.TEXT_MATCH)) {
+ // TODO: fix it with right logic
+ return true;
+ }
if (mapMeta.getIndexedColumns().size() == 0 || columnExpressions.size() == 0) {
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index e57a841..ab339e8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.datamap;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -144,7 +145,7 @@ public final class DataMapStoreManager {
* The datamap is created using datamap name, datamap factory class and table identifier.
*/
private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
- DataMapSchema dataMapSchema) throws MalformedDataMapCommandException {
+ DataMapSchema dataMapSchema) throws MalformedDataMapCommandException, IOException {
DataMapFactory dataMapFactory;
try {
// try to create datamap by reflection to test whether it is a valid DataMapFactory class
@@ -162,7 +163,7 @@ public final class DataMapStoreManager {
}
public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier,
- DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) {
+ DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) throws IOException {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
index f036b0b..9fbdd90 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
@@ -42,7 +42,7 @@ public interface DataMap<T extends Blocklet> {
* blocklets where these filters can exist.
*/
List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions);
+ List<PartitionSpec> partitions) throws IOException;
// TODO Move this method to Abstract class
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index 7fe910e..f2c376a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -35,7 +35,7 @@ public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
*/
- void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
+ void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) throws IOException;
/**
* Return a new write for this datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
index 4911f7b..6a3ee18 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapWriter.java
@@ -54,12 +54,12 @@ public abstract class DataMapWriter {
*
* @param blockId file name of the carbondata file
*/
- public abstract void onBlockStart(String blockId);
+ public abstract void onBlockStart(String blockId) throws IOException;
/**
* End of block notification
*/
- public abstract void onBlockEnd(String blockId);
+ public abstract void onBlockEnd(String blockId) throws IOException;
/**
* Start of new blocklet notification.
@@ -81,7 +81,8 @@ public abstract class DataMapWriter {
* Implementation should copy the content of `pages` as needed, because `pages` memory
* may be freed after this method returns, if using unsafe column page.
*/
- public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages);
+ public abstract void onPageAdded(int blockletId, int pageId, ColumnPage[] pages)
+ throws IOException;
/**
* This is called during closing of writer.So after this call no more data will be sent to this
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
index 4d20cdb..f9fdafb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/CoarseGrainDataMapFactory.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.carbondata.core.datamap.dev.cgdatamap;
import org.apache.carbondata.common.annotations.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
index 831acc8..d66c5b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
@@ -42,5 +42,6 @@ public enum ExpressionType {
TRUE,
STARTSWITH,
ENDSWITH,
- CONTAINSWITH
+ CONTAINSWITH,
+ TEXT_MATCH
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml
new file mode 100644
index 0000000..ee504c6
--- /dev/null
+++ b/datamap/lucene/pom.xml
@@ -0,0 +1,149 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-lucene</artifactId>
+ <name>Apache CarbonData :: Lucene Index DataMap</name>
+
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ <lucene.version>6.3.0</lucene.version>
+ <solr.version>6.3.0</solr.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-sandbox</artifactId>
+ <version>${lucene.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-core</artifactId>
+ <version>${solr.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <testSourceDirectory>src/test/scala</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/resources</directory>
+ </resource>
+ <resource>
+ <directory>.</directory>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <id>testCompile</id>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <phase>test</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
new file mode 100644
index 0000000..0b7df86
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.store.hdfs.HdfsDirectory;
+
+@InterfaceAudience.Internal
+public class LuceneCoarseGrainDataMap extends CoarseGrainDataMap {
+
+ /**
+ * log information
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LuceneCoarseGrainDataMap.class.getName());
+
+ public static final int BLOCKID_ID = 0;
+
+ public static final int BLOCKLETID_ID = 1;
+
+ public static final int PAGEID_ID = 2;
+
+ public static final int ROWID_ID = 3;
+ /**
+ * searcher object for this datamap
+ */
+ private IndexSearcher indexSearcher = null;
+
+ /**
+ * default max values to return
+ */
+ private static int MAX_RESULT_NUMBER = 100;
+
+ /**
+ * analyzer for lucene index
+ */
+ private Analyzer analyzer;
+
+ LuceneCoarseGrainDataMap(Analyzer analyzer) {
+ this.analyzer = analyzer;
+ }
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ @Override
+ public void init(DataMapModel dataMapModel) throws MemoryException, IOException {
+ // get this path from file path
+ Path indexPath = FileFactory.getPath(dataMapModel.getFilePath());
+
+ LOGGER.info("Lucene index read path " + indexPath.toString());
+
+ // get file system , use hdfs file system , realized in solr project
+ FileSystem fs = FileFactory.getFileSystem(indexPath);
+
+ // check this path valid
+ if (!fs.exists(indexPath)) {
+ String errorMessage = String.format("index directory %s not exists.", indexPath);
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ if (!fs.isDirectory(indexPath)) {
+ String errorMessage = String.format("error index path %s, must be directory", indexPath);
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ // open this index path , use HDFS default configuration
+ Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration());
+
+ IndexReader indexReader = DirectoryReader.open(indexDir);
+ if (indexReader == null) {
+ throw new RuntimeException("failed to create index reader object");
+ }
+
+ // create a index searcher object
+ indexSearcher = new IndexSearcher(indexReader);
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ */
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<String> partitions) throws IOException {
+
+ // convert filter expr into lucene list query
+ List<String> fields = new ArrayList<String>();
+
+ // only for test , query all data
+ String strQuery = "*:*";
+
+ String[] sFields = new String[fields.size()];
+ fields.toArray(sFields);
+
+ // get analyzer
+ if (analyzer == null) {
+ analyzer = new StandardAnalyzer();
+ }
+
+ // use MultiFieldQueryParser to parser query
+ QueryParser queryParser = new MultiFieldQueryParser(sFields, analyzer);
+ Query query;
+ try {
+ query = queryParser.parse(strQuery);
+ } catch (ParseException e) {
+ String errorMessage = String
+ .format("failed to filter block with query %s, detail is %s", strQuery, e.getMessage());
+ LOGGER.error(errorMessage);
+ return null;
+ }
+
+ // execute index search
+ TopDocs result;
+ try {
+ result = indexSearcher.search(query, MAX_RESULT_NUMBER);
+ } catch (IOException e) {
+ String errorMessage =
+ String.format("failed to search lucene data, detail is %s", e.getMessage());
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ // temporary data, delete duplicated data
+ // Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
+ Map<String, Set<Number>> mapBlocks = new HashMap<String, Set<Number>>();
+
+ for (ScoreDoc scoreDoc : result.scoreDocs) {
+ // get a document
+ Document doc = indexSearcher.doc(scoreDoc.doc);
+
+ // get all fields
+ List<IndexableField> fieldsInDoc = doc.getFields();
+
+ // get this block id Map<BlockId, Set<BlockletId>>>>
+ String blockId = fieldsInDoc.get(BLOCKID_ID).stringValue();
+ Set<Number> setBlocklets = mapBlocks.get(blockId);
+ if (setBlocklets == null) {
+ setBlocklets = new HashSet<Number>();
+ mapBlocks.put(blockId, setBlocklets);
+ }
+
+ // get the blocklet id Set<BlockletId>
+ Number blockletId = fieldsInDoc.get(BLOCKLETID_ID).numericValue();
+ if (!setBlocklets.contains(blockletId.intValue())) {
+ setBlocklets.add(blockletId.intValue());
+ }
+ }
+
+ // result blocklets
+ List<Blocklet> blocklets = new ArrayList<Blocklet>();
+
+ // transform all blocks into result type blocklets Map<BlockId, Set<BlockletId>>
+ for (Map.Entry<String, Set<Number>> mapBlock : mapBlocks.entrySet()) {
+ String blockId = mapBlock.getKey();
+ Set<Number> setBlocklets = mapBlock.getValue();
+
+ // for blocklets in this block Set<BlockletId>
+ for (Number blockletId : setBlocklets) {
+
+ // add a CoarseGrainBlocklet
+ blocklets.add(new Blocklet(blockId, blockletId.toString()));
+ }
+ }
+
+ return blocklets;
+ }
+
+ @Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ return true;
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ @Override
+ public void clear() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
new file mode 100644
index 0000000..7f9cc1c
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.memory.MemoryException;
+
+/**
+ * FG level of lucene DataMap
+ */
+@InterfaceAudience.Internal
+public class LuceneCoarseGrainDataMapFactory extends LuceneDataMapFactoryBase<CoarseGrainDataMap> {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LuceneCoarseGrainDataMapFactory.class.getName());
+
+ /**
+ * Get the datamap for segmentid
+ */
+ public List<CoarseGrainDataMap> getDataMaps(String segmentId) throws IOException {
+ List<CoarseGrainDataMap> lstDataMap = new ArrayList<>();
+ CoarseGrainDataMap dataMap = new LuceneCoarseGrainDataMap(analyzer);
+ try {
+ dataMap.init(new DataMapModel(
+ LuceneDataMapWriter.genDataMapStorePath(
+ tableIdentifier.getTablePath(), segmentId, dataMapName)));
+ } catch (MemoryException e) {
+ LOGGER.error("failed to get lucene datamap , detail is {}" + e.getMessage());
+ return lstDataMap;
+ }
+ lstDataMap.add(dataMap);
+ return lstDataMap;
+ }
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
+ return getDataMaps(distributable.getSegmentId());
+ }
+
+ @Override
+ public DataMapLevel getDataMapType() {
+ return DataMapLevel.CG;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapDistributable.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapDistributable.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapDistributable.java
new file mode 100644
index 0000000..19e4035
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapDistributable.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+
+@InterfaceAudience.Internal
+class LuceneDataMapDistributable extends DataMapDistributable {
+
+ // TODO: seems no one use this?
+ private String dataPath;
+
+ LuceneDataMapDistributable(String dataPath) {
+ this.dataPath = dataPath;
+ }
+
+ public String getDataPath() {
+ return dataPath;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
new file mode 100644
index 0000000..5eb7054
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.dev.DataMap;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+
+/**
+ * Base implementation for CG and FG lucene DataMapFactory.
+ */
+@InterfaceAudience.Internal
+abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFactory<T> {
+
+ /**
+ * Logger
+ */
+ final LogService LOGGER = LogServiceFactory.getLogService(this.getClass().getName());
+
+ /**
+ * table's index columns
+ */
+ DataMapMeta dataMapMeta = null;
+
+ /**
+ * analyzer for lucene
+ */
+ Analyzer analyzer = null;
+
+ /**
+ * index name
+ */
+ String dataMapName = null;
+
+ /**
+ * table identifier
+ */
+ AbsoluteTableIdentifier tableIdentifier = null;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema)
+ throws IOException {
+ Objects.requireNonNull(identifier);
+ Objects.requireNonNull(dataMapSchema);
+
+ this.tableIdentifier = identifier;
+ this.dataMapName = dataMapSchema.getDataMapName();
+
+ // get carbonmetadata from carbonmetadata instance
+ CarbonMetadata carbonMetadata = CarbonMetadata.getInstance();
+
+ String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName();
+
+ // get carbon table
+ CarbonTable carbonTable = carbonMetadata.getCarbonTable(tableUniqueName);
+ if (carbonTable == null) {
+ String errorMessage =
+ String.format("failed to get carbon table with name %s", tableUniqueName);
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ TableInfo tableInfo = carbonTable.getTableInfo();
+ List<ColumnSchema> lstCoumnSchemas = tableInfo.getFactTable().getListOfColumns();
+
+ // currently add all columns into lucene indexer
+ // TODO:only add index columns
+ List<String> indexedColumns = new ArrayList<String>();
+ for (ColumnSchema columnSchema : lstCoumnSchemas) {
+ if (!columnSchema.isInvisible()) {
+ indexedColumns.add(columnSchema.getColumnName());
+ }
+ }
+
+ // get indexed columns
+ // Map<String, String> properties = dataMapSchema.getProperties();
+ // String columns = properties.get("text_column");
+ // if (columns != null) {
+ // String[] columnArray = columns.split(CarbonCommonConstants.COMMA, -1);
+ // Collections.addAll(indexedColumns, columnArray);
+ // }
+
+ // add optimizedOperations
+ List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>();
+ // optimizedOperations.add(ExpressionType.EQUALS);
+ // optimizedOperations.add(ExpressionType.GREATERTHAN);
+ // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO);
+ // optimizedOperations.add(ExpressionType.LESSTHAN);
+ // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO);
+ // optimizedOperations.add(ExpressionType.NOT);
+ optimizedOperations.add(ExpressionType.TEXT_MATCH);
+ this.dataMapMeta = new DataMapMeta(indexedColumns, optimizedOperations);
+
+ // get analyzer
+ // TODO: how to get analyzer ?
+ analyzer = new StandardAnalyzer();
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ public DataMapWriter createWriter(String segmentId, String writeDirectoryPath) {
+ LOGGER.info("lucene data write to " + writeDirectoryPath);
+ return new LuceneDataMapWriter(
+ tableIdentifier, dataMapName, segmentId, writeDirectoryPath, true);
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ */
+ public List<DataMapDistributable> toDistributable(String segmentId) {
+ List<DataMapDistributable> lstDataMapDistribute = new ArrayList<DataMapDistributable>();
+ DataMapDistributable luceneDataMapDistributable = new LuceneDataMapDistributable(
+ CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segmentId));
+ lstDataMapDistribute.add(luceneDataMapDistributable);
+ return lstDataMapDistribute;
+ }
+
+ public void fireEvent(Event event) {
+
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ public void clear(String segmentId) {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ public void clear() {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ public DataMapMeta getMeta() {
+ return dataMapMeta;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
new file mode 100644
index 0000000..849fc2e
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoublePoint;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.IntRangeField;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.solr.store.hdfs.HdfsDirectory;
+
+/**
+ * Implementation to write lucene index while loading
+ */
+@InterfaceAudience.Internal
+public class LuceneDataMapWriter extends DataMapWriter {
+ /**
+ * logger
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LuceneDataMapWriter.class.getName());
+
+ /**
+ * index writer
+ */
+ private IndexWriter indexWriter = null;
+
+ private Analyzer analyzer = null;
+
+ private String blockId = null;
+
+ private String dataMapName = null;
+
+ private boolean isFineGrain = true;
+
+ private static final String BLOCKID_NAME = "blockId";
+
+ private static final String BLOCKLETID_NAME = "blockletId";
+
+ private static final String PAGEID_NAME = "pageId";
+
+ private static final String ROWID_NAME = "rowId";
+
+ LuceneDataMapWriter(AbsoluteTableIdentifier identifier, String dataMapName, String segmentId,
+ String writeDirectoryPath, boolean isFineGrain) {
+ super(identifier, segmentId, writeDirectoryPath);
+ this.dataMapName = dataMapName;
+ this.isFineGrain = isFineGrain;
+ }
+
+ private String getIndexPath() {
+ if (isFineGrain) {
+ return genDataMapStorePath(identifier.getTablePath(), segmentId, dataMapName);
+ } else {
+ // TODO: where write data in coarse grain data map
+ return genDataMapStorePath(identifier.getTablePath(), segmentId, dataMapName);
+ }
+ }
+
+ /**
+ * Start of new block notification.
+ */
+ public void onBlockStart(String blockId) throws IOException {
+ // save this block id for lucene index , used in onPageAdd function
+ this.blockId = blockId;
+
+ // get index path, put index data into segment's path
+ String strIndexPath = getIndexPath();
+ Path indexPath = FileFactory.getPath(strIndexPath);
+ FileSystem fs = FileFactory.getFileSystem(indexPath);
+
+ // if index path not exists, create it
+ if (fs.exists(indexPath)) {
+ fs.mkdirs(indexPath);
+ }
+
+ if (null == analyzer) {
+ analyzer = new StandardAnalyzer();
+ }
+
+ // create a index writer
+ Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration());
+ indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer));
+
+ }
+
+ /**
+ * End of block notification
+ */
+ public void onBlockEnd(String blockId) throws IOException {
+ // clean this block id
+ this.blockId = null;
+
+ // finished a file , close this index writer
+ if (indexWriter != null) {
+ indexWriter.close();
+ }
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ */
+ public void onBlockletStart(int blockletId) {
+
+ }
+
+ /**
+ * End of blocklet notification
+ */
+ public void onBlockletEnd(int blockletId) {
+
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ public void onPageAdded(int blockletId, int pageId, ColumnPage[] pages) throws IOException {
+ // save index data into ram, write into disk after one page finished
+ RAMDirectory ramDir = new RAMDirectory();
+ IndexWriter ramIndexWriter = new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
+
+ int columnsCount = pages.length;
+ if (columnsCount <= 0) {
+ LOGGER.warn("empty data");
+ ramIndexWriter.close();
+ ramDir.close();
+ return;
+ }
+ int pageSize = pages[0].getPageSize();
+ for (int rowId = 0; rowId < pageSize; rowId++) {
+ // create a new document
+ Document doc = new Document();
+
+ // add block id, save this id
+ doc.add(new StringField(BLOCKID_NAME, blockId, Field.Store.YES));
+
+ // add blocklet Id
+ doc.add(new IntPoint(BLOCKLETID_NAME, new int[] { blockletId }));
+ doc.add(new StoredField(BLOCKLETID_NAME, blockletId));
+ //doc.add(new NumericDocValuesField(BLOCKLETID_NAME,blockletId));
+
+ // add page id and row id in Fine Grain data map
+ if (isFineGrain) {
+ // add page Id
+ doc.add(new IntPoint(PAGEID_NAME, new int[] { pageId }));
+ doc.add(new StoredField(PAGEID_NAME, pageId));
+ //doc.add(new NumericDocValuesField(PAGEID_NAME,pageId));
+
+ // add row id
+ doc.add(new IntPoint(ROWID_NAME, new int[] { rowId }));
+ doc.add(new StoredField(ROWID_NAME, rowId));
+ //doc.add(new NumericDocValuesField(ROWID_NAME,rowId));
+ }
+
+ // add other fields
+ for (int colIdx = 0; colIdx < columnsCount; colIdx++) {
+ if (!pages[colIdx].getNullBits().get(rowId)) {
+ addField(doc, pages[colIdx], rowId, Field.Store.NO);
+ }
+ }
+
+ // add this document
+ ramIndexWriter.addDocument(doc);
+
+ }
+ // close ram writer
+ ramIndexWriter.close();
+
+ // add ram index data into disk
+ indexWriter.addIndexes(new Directory[] { ramDir });
+
+ // delete this ram data
+ ramDir.close();
+ }
+
+ private boolean addField(Document doc, ColumnPage page, int rowId, Field.Store store) {
+ //get field name
+ String fieldName = page.getColumnSpec().getFieldName();
+
+ //get field type
+ DataType type = page.getDataType();
+
+ if (type == DataTypes.BYTE) {
+ // byte type , use int range to deal with byte, lucene has no byte type
+ byte value = page.getByte(rowId);
+ IntRangeField field =
+ new IntRangeField(fieldName, new int[] { Byte.MIN_VALUE }, new int[] { Byte.MAX_VALUE });
+ field.setIntValue(value);
+ doc.add(field);
+
+ // if need store it , add StoredField
+ if (store == Field.Store.YES) {
+ doc.add(new StoredField(fieldName, (int) value));
+ }
+ } else if (type == DataTypes.SHORT) {
+ // short type , use int range to deal with short type, lucene has no short type
+ short value = page.getShort(rowId);
+ IntRangeField field = new IntRangeField(fieldName, new int[] { Short.MIN_VALUE },
+ new int[] { Short.MAX_VALUE });
+ field.setShortValue(value);
+ doc.add(field);
+
+ // if need store it , add StoredField
+ if (store == Field.Store.YES) {
+ doc.add(new StoredField(fieldName, (int) value));
+ }
+ } else if (type == DataTypes.INT) {
+ // int type , use int point to deal with int type
+ int value = page.getInt(rowId);
+ doc.add(new IntPoint(fieldName, new int[] { value }));
+
+ // if need store it , add StoredField
+ if (store == Field.Store.YES) {
+ doc.add(new StoredField(fieldName, value));
+ }
+ } else if (type == DataTypes.LONG) {
+ // long type , use long point to deal with long type
+ long value = page.getLong(rowId);
+ doc.add(new LongPoint(fieldName, new long[] { value }));
+
+ // if need store it , add StoredField
+ if (store == Field.Store.YES) {
+ doc.add(new StoredField(fieldName, value));
+ }
+ } else if (type == DataTypes.FLOAT) {
+ float value = page.getFloat(rowId);
+ doc.add(new FloatPoint(fieldName, new float[] { value }));
+ if (store == Field.Store.YES) {
+ doc.add(new FloatPoint(fieldName, value));
+ }
+ } else if (type == DataTypes.DOUBLE) {
+ double value = page.getDouble(rowId);
+ doc.add(new DoublePoint(fieldName, new double[] { value }));
+ if (store == Field.Store.YES) {
+ doc.add(new DoublePoint(fieldName, value));
+ }
+ } else if (type == DataTypes.STRING) {
+ byte[] value = page.getBytes(rowId);
+ // TODO: how to get string value
+ String strValue = null;
+ try {
+ strValue = new String(value, 2, value.length - 2, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ doc.add(new TextField(fieldName, strValue, store));
+ } else if (type == DataTypes.DATE) {
+ // TODO: how to get data value
+ } else if (type == DataTypes.TIMESTAMP) {
+ // TODO: how to get
+ } else if (type == DataTypes.BOOLEAN) {
+ boolean value = page.getBoolean(rowId);
+ IntRangeField field = new IntRangeField(fieldName, new int[] { 0 }, new int[] { 1 });
+ field.setIntValue(value ? 1 : 0);
+ doc.add(field);
+ if (store == Field.Store.YES) {
+ doc.add(new StoredField(fieldName, value ? 1 : 0));
+ }
+ } else {
+ LOGGER.error("unsupport data type " + type);
+ throw new RuntimeException("unsupported data type " + type);
+ }
+ return true;
+ }
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ public void finish() throws IOException {
+
+ }
+
+ /**
+ * Return store path for datamap
+ */
+ static String genDataMapStorePath(String tablePath, String segmentId, String dataMapName) {
+ return CarbonTablePath.getSegmentPath(tablePath, segmentId) + File.separator + dataMapName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
new file mode 100644
index 0000000..c649545
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainDataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.store.hdfs.HdfsDirectory;
+
+@InterfaceAudience.Internal
+public class LuceneFineGrainDataMap extends FineGrainDataMap {
+
+ private static final int BLOCKID_ID = 0;
+
+ private static final int BLOCKLETID_ID = 1;
+
+ private static final int PAGEID_ID = 2;
+
+ private static final int ROWID_ID = 3;
+
+ /**
+ * log information
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LuceneFineGrainDataMap.class.getName());
+
+ /**
+ * searcher object for this datamap
+ */
+ private IndexSearcher indexSearcher = null;
+
+ /**
+ * default max values to return
+ */
+ private static int MAX_RESULT_NUMBER = 100;
+
+ /**
+ * analyzer for lucene index
+ */
+ private Analyzer analyzer;
+
+ LuceneFineGrainDataMap(Analyzer analyzer) {
+ this.analyzer = analyzer;
+ }
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ public void init(DataMapModel dataMapModel) throws MemoryException, IOException {
+ // get this path from file path
+ Path indexPath = FileFactory.getPath(dataMapModel.getFilePath());
+
+ LOGGER.info("Lucene index read path " + indexPath.toString());
+
+ // get file system , use hdfs file system , realized in solr project
+ FileSystem fs = FileFactory.getFileSystem(indexPath);
+
+ // check this path valid
+ if (!fs.exists(indexPath)) {
+ String errorMessage = String.format("index directory %s not exists.", indexPath);
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ if (!fs.isDirectory(indexPath)) {
+ String errorMessage = String.format("error index path %s, must be directory", indexPath);
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ // open this index path , use HDFS default configuration
+ Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration());
+
+ IndexReader indexReader = DirectoryReader.open(indexDir);
+ if (indexReader == null) {
+ throw new RuntimeException("failed to create index reader object");
+ }
+
+ // create a index searcher object
+ indexSearcher = new IndexSearcher(indexReader);
+ }
+
+ /**
+ * Return the query string in the first TEXT_MATCH expression in the expression tree
+ */
+ private String getQueryString(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.TEXT_MATCH) {
+ return expression.getString();
+ }
+
+ for (Expression child : expression.getChildren()) {
+ String queryString = getQueryString(child);
+ if (queryString != null) {
+ return queryString;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ */
+ @Override
+ public List<FineGrainBlocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties, List<String> partitions) throws IOException {
+
+ // convert filter expr into lucene list query
+ List<String> fields = new ArrayList<String>();
+
+ // only for test , query all data
+ String strQuery = getQueryString(filterExp.getFilterExpression());
+
+ String[] sFields = new String[fields.size()];
+ fields.toArray(sFields);
+
+ // get analyzer
+ if (analyzer == null) {
+ analyzer = new StandardAnalyzer();
+ }
+
+ // use MultiFieldQueryParser to parser query
+ QueryParser queryParser = new MultiFieldQueryParser(sFields, analyzer);
+ Query query;
+ try {
+ query = queryParser.parse(strQuery);
+ } catch (ParseException e) {
+ String errorMessage = String.format(
+ "failed to filter block with query %s, detail is %s", strQuery, e.getMessage());
+ LOGGER.error(errorMessage);
+ return null;
+ }
+
+ // execute index search
+ TopDocs result;
+ try {
+ result = indexSearcher.search(query, MAX_RESULT_NUMBER);
+ } catch (IOException e) {
+ String errorMessage =
+ String.format("failed to search lucene data, detail is %s", e.getMessage());
+ LOGGER.error(errorMessage);
+ throw new IOException(errorMessage);
+ }
+
+ // temporary data, delete duplicated data
+ // Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
+ Map<String, Map<String, Map<Integer, Set<Integer>>>> mapBlocks = new HashMap<>();
+
+ for (ScoreDoc scoreDoc : result.scoreDocs) {
+ // get a document
+ Document doc = indexSearcher.doc(scoreDoc.doc);
+
+ // get all fields
+ List<IndexableField> fieldsInDoc = doc.getFields();
+
+ // get this block id Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
+ String blockId = fieldsInDoc.get(BLOCKID_ID).stringValue();
+ Map<String, Map<Integer, Set<Integer>>> mapBlocklets = mapBlocks.get(blockId);
+ if (mapBlocklets == null) {
+ mapBlocklets = new HashMap<>();
+ mapBlocks.put(blockId, mapBlocklets);
+ }
+
+ // get the blocklet id Map<BlockletId, Map<PageId, Set<RowId>>>
+ String blockletId = fieldsInDoc.get(BLOCKLETID_ID).stringValue();
+ Map<Integer, Set<Integer>> mapPageIds = mapBlocklets.get(blockletId);
+ if (mapPageIds == null) {
+ mapPageIds = new HashMap<>();
+ mapBlocklets.put(blockletId, mapPageIds);
+ }
+
+ // get the page id Map<PageId, Set<RowId>>
+ Number pageId = fieldsInDoc.get(PAGEID_ID).numericValue();
+ Set<Integer> setRowId = mapPageIds.get(pageId.intValue());
+ if (setRowId == null) {
+ setRowId = new HashSet<>();
+ mapPageIds.put(pageId.intValue(), setRowId);
+ }
+
+ // get the row id Set<RowId>
+ Number rowId = fieldsInDoc.get(ROWID_ID).numericValue();
+ setRowId.add(rowId.intValue());
+ }
+
+ // result blocklets
+ List<FineGrainBlocklet> blocklets = new ArrayList<>();
+
+ // transform all blocks into result type blocklets
+ // Map<BlockId, Map<BlockletId, Map<PageId, Set<RowId>>>>
+ for (Map.Entry<String, Map<String, Map<Integer, Set<Integer>>>> mapBlock :
+ mapBlocks.entrySet()) {
+ String blockId = mapBlock.getKey();
+ Map<String, Map<Integer, Set<Integer>>> mapBlocklets = mapBlock.getValue();
+ // for blocklets in this block Map<BlockletId, Map<PageId, Set<RowId>>>
+ for (Map.Entry<String, Map<Integer, Set<Integer>>> mapBlocklet : mapBlocklets.entrySet()) {
+ String blockletId = mapBlocklet.getKey();
+ Map<Integer, Set<Integer>> mapPageIds = mapBlocklet.getValue();
+ List<FineGrainBlocklet.Page> pages = new ArrayList<FineGrainBlocklet.Page>();
+
+ // for pages in this blocklet Map<PageId, Set<RowId>>>
+ for (Map.Entry<Integer, Set<Integer>> mapPageId : mapPageIds.entrySet()) {
+ // construct array rowid
+ int[] rowIds = new int[mapPageId.getValue().size()];
+ int i = 0;
+ // for rowids in this page Set<RowId>
+ for (Integer rowid : mapPageId.getValue()) {
+ rowIds[i++] = rowid;
+ }
+ // construct one page
+ FineGrainBlocklet.Page page = new FineGrainBlocklet.Page();
+ page.setPageId(mapPageId.getKey());
+ page.setRowId(rowIds);
+
+ // add this page into list pages
+ pages.add(page);
+ }
+
+ // add a FineGrainBlocklet
+ blocklets.add(new FineGrainBlocklet(blockId, blockletId, pages));
+ }
+ }
+
+ return blocklets;
+ }
+
+ @Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ return true;
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ @Override
+ public void clear() {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
new file mode 100644
index 0000000..e35d5bf
--- /dev/null
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainDataMap;
+import org.apache.carbondata.core.memory.MemoryException;
+
+/**
+ * CG level of lucene DataMap
+ */
+@InterfaceAudience.Internal
+public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<FineGrainDataMap> {
+
+ /**
+ * Get the datamap for segmentid
+ */
+ public List<FineGrainDataMap> getDataMaps(String segmentId) throws IOException {
+ List<FineGrainDataMap> lstDataMap = new ArrayList<>();
+ FineGrainDataMap dataMap = new LuceneFineGrainDataMap(analyzer);
+ try {
+ dataMap.init(new DataMapModel(
+ LuceneDataMapWriter.genDataMapStorePath(
+ tableIdentifier.getTablePath(), segmentId, dataMapName)));
+ } catch (MemoryException e) {
+ LOGGER.error("failed to get lucene datamap , detail is {}" + e.getMessage());
+ return lstDataMap;
+ }
+ lstDataMap.add(dataMap);
+ return lstDataMap;
+ }
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ public List<FineGrainDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
+ return getDataMaps(distributable.getSegmentId());
+ }
+
+ @Override
+ public DataMapLevel getDataMapType() {
+ return DataMapLevel.FG;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapSuite.scala
new file mode 100644
index 0000000..a461f04
--- /dev/null
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+
+class LuceneCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/datamap_input.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 15000
+ LuceneFineGrainDataMapSuite.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test lucene coarse grain data map") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE DATAMAP dm ON TABLE datamap_test
+ | USING '${classOf[LuceneCoarseGrainDataMapFactory].getName}'
+ """.stripMargin)
+
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ LuceneFineGrainDataMapSuite.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
new file mode 100644
index 0000000..4766281
--- /dev/null
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.lucene
+
+import java.io.{File, PrintWriter}
+
+import scala.util.Random
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.metadata.CarbonMetadata
+
+class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/datamap_input.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 15000
+ LuceneFineGrainDataMapSuite.createFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test lucene fine grain data map") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | CREATE DATAMAP dm ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ """.stripMargin)
+
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+
+ sql("SELECT * FROM datamap_test ORDER BY id").show
+
+ // sql("select * from normal_test where name='n34000'").show
+ sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10')").show
+ sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('name:n10*')").show
+ sql("SELECT * FROM datamap_test WHERE TEXT_MATCH('city:c020')").show
+
+ // checkAnswer(
+ // sql("select * from datamap_test where match('name:n34000')"),
+ // sql("select * from normal_test where name='n34000'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ LuceneFineGrainDataMapSuite.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
+
+object LuceneFineGrainDataMapSuite {
+ def createFile(fileName: String, line: Int = 10000, start: Int = 0) = {
+ val write = new PrintWriter(new File(fileName))
+ for (i <- start until (start + line)) {
+ write.println(i + "," + "n" + i + "," + "c0" + i + "," + Random.nextInt(80))
+ }
+ write.close()
+ }
+
+ def deleteFile(fileName: String): Unit = {
+ val file = new File(fileName)
+ if (file.exists()) {
+ file.delete()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark-common-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml
index d1c04ae..b7f19fd 100644
--- a/integration/spark-common-test/pom.xml
+++ b/integration/spark-common-test/pom.xml
@@ -99,6 +99,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-lucene</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 977b31d..fae87a5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -246,7 +246,7 @@ class FGDataMap extends FineGrainDataMap {
* Clear complete index table and release memory.
*/
override def clear():Unit = {
- ???
+
}
override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
index 10955a3..ea571d7 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.datamap;
+import java.io.IOException;
+
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -62,7 +64,7 @@ public interface DataMapProvider {
* Implementation should initialize metadata for datamap, like creating table
*/
void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
- SparkSession sparkSession) throws MalformedDataMapCommandException;
+ SparkSession sparkSession) throws MalformedDataMapCommandException, IOException;
/**
* Initialize a datamap's data.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
index 2a6a70a..c7651bb 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
@@ -17,6 +17,8 @@
package org.apache.carbondata.datamap;
+import java.io.IOException;
+
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.exceptions.MetadataProcessException;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
@@ -37,7 +39,7 @@ public class IndexDataMapProvider implements DataMapProvider {
@Override
public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
- SparkSession sparkSession) throws MalformedDataMapCommandException {
+ SparkSession sparkSession) throws MalformedDataMapCommandException, IOException {
DataMapFactory dataMapFactory = createIndexDataMapFactory(dataMapSchema);
DataMapStoreManager.getInstance().registerDataMap(
mainTable.getAbsoluteTableIdentifier(), dataMapSchema, dataMapFactory);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
new file mode 100644
index 0000000..fceb729
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/expression/MatchExpression.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.expression;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+@InterfaceAudience.Internal
+public class MatchExpression extends Expression {
+ private String queryString;
+
+ public MatchExpression(String queryString) {
+ this.queryString = queryString;
+ }
+
+ @Override
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return null;
+ }
+
+ @Override
+ public ExpressionType getFilterExpressionType() {
+ return ExpressionType.TEXT_MATCH;
+ }
+
+ @Override
+ public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+
+ }
+
+ @Override
+ public String getString() {
+ return queryString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/scala/org/apache/carbondata/datamap/TextMatchUDF.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/TextMatchUDF.scala b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/TextMatchUDF.scala
new file mode 100644
index 0000000..093e479
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/TextMatchUDF.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap
+
+import org.apache.spark.sql.sources.Filter
+
+import org.apache.carbondata.common.annotations.InterfaceAudience
+
+@InterfaceAudience.Internal
+class TextMatchUDF extends ((String) => Boolean) with Serializable {
+ override def apply(v1: String): Boolean = {
+ v1.length > 0
+ }
+}
+
+@InterfaceAudience.Internal
+case class TextMatch(queryString: String) extends Filter {
+ override def references: Array[String] = null
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 9315208..8c3ca0f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util._
+import org.apache.carbondata.datamap.TextMatchUDF
import org.apache.carbondata.events._
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostStatusUpdateEvent, LoadTablePreExecutionEvent, LoadTablePreStatusUpdateEvent}
import org.apache.carbondata.spark.rdd.SparkReadSupport
@@ -66,6 +67,10 @@ class CarbonEnv {
// column to sum and count.
sparkSession.udf.register("preAggLoad", () => "")
+ // register for lucene datamap
+ // TODO: move it to proper place, it should be registered by datamap implementation
+ sparkSession.udf.register("text_match", new TextMatchUDF)
+
// added for handling timeseries function like hour, minute, day , month , year
sparkSession.udf.register("timeseries", new TimeSeriesFunction)
// acquiring global level lock so global configuration will be updated by only one thread
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index e82c9d7..47da9a5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -35,12 +35,14 @@ import org.apache.spark.sql.sources.{BaseRelation, Filter}
import org.apache.spark.sql.types._
import org.apache.spark.sql.CarbonExpressions.{MatchCast => Cast}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.datamap.{TextMatch, TextMatchUDF}
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -527,6 +529,13 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
*/
protected[sql] def translateFilter(predicate: Expression, or: Boolean = false): Option[Filter] = {
predicate match {
+ case u: ScalaUDF if u.function.isInstanceOf[TextMatchUDF] =>
+ if (u.children.size > 1) {
+ throw new MalformedCarbonCommandException(
+ "TEXT_MATCH UDF syntax: TEXT_MATCH('luceneQuerySyntax')")
+ }
+ Some(TextMatch(u.children.head.toString()))
+
case or@Or(left, right) =>
val leftFilter = translateFilter(left, true)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 38c5146..0aedd40 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -45,6 +45,8 @@ import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.datamap.TextMatch
+import org.apache.carbondata.datamap.expression.MatchExpression
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -140,6 +142,8 @@ object CarbonFilters {
Some(transformExpression(expr))
case FalseExpr() =>
Some(new FalseExpression(null))
+ case TextMatch(queryString) =>
+ Some(new MatchExpression(queryString))
case _ => None
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0891972..fdc9210 100644
--- a/pom.xml
+++ b/pom.xml
@@ -342,6 +342,7 @@
<artifactId>findbugs-maven-plugin</artifactId>
<version>3.0.4</version>
<configuration>
+ <skip>true</skip>
<excludeFilterFile>${dev.path}/findbugs-exclude.xml</excludeFilterFile>
<failOnError>true</failOnError>
<findbugsXmlOutput>true</findbugsXmlOutput>
@@ -481,6 +482,7 @@
<module>streaming</module>
<module>examples/spark2</module>
<module>datamap/examples</module>
+ <module>datamap/lucene</module>
</modules>
<build>
<plugins>
@@ -535,6 +537,7 @@
<module>integration/presto</module>
<module>streaming</module>
<module>examples/spark2</module>
+ <module>datamap/lucene</module>
</modules>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 22a273b..2e39c91 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -85,7 +85,7 @@ public class DataMapWriterListener {
LOG.info("DataMapWriter " + writer + " added");
}
- public void onBlockStart(String blockId, String blockPath) {
+ public void onBlockStart(String blockId, String blockPath) throws IOException {
for (List<DataMapWriter> writers : registry.values()) {
for (DataMapWriter writer : writers) {
writer.onBlockStart(blockId);
@@ -93,7 +93,7 @@ public class DataMapWriterListener {
}
}
- public void onBlockEnd(String blockId) {
+ public void onBlockEnd(String blockId) throws IOException {
for (List<DataMapWriter> writers : registry.values()) {
for (DataMapWriter writer : writers) {
writer.onBlockEnd(blockId);
@@ -123,7 +123,7 @@ public class DataMapWriterListener {
* @param pageId sequence number of page, start from 0
* @param tablePage page data
*/
- public void onPageAdded(int blockletId, int pageId, TablePage tablePage) {
+ public void onPageAdded(int blockletId, int pageId, TablePage tablePage) throws IOException {
Set<Map.Entry<List<String>, List<DataMapWriter>>> entries = registry.entrySet();
for (Map.Entry<List<String>, List<DataMapWriter>> entry : entries) {
List<String> indexedColumns = entry.getKey();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 8d26ad2..4064c0d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -228,13 +228,21 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter {
private void notifyDataMapBlockStart() {
if (listener != null) {
- listener.onBlockStart(carbonDataFileName, constructFactFileFullPath());
+ try {
+ listener.onBlockStart(carbonDataFileName, constructFactFileFullPath());
+ } catch (IOException e) {
+ throw new CarbonDataWriterException("Problem while writing datamap", e);
+ }
}
}
private void notifyDataMapBlockEnd() {
if (listener != null) {
- listener.onBlockEnd(carbonDataFileName);
+ try {
+ listener.onBlockEnd(carbonDataFileName);
+ } catch (IOException e) {
+ throw new CarbonDataWriterException("Problem while writing datamap", e);
+ }
}
blockletId = 0;
}
[12/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
deleted file mode 100644
index 6629d31..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to fixed length dimension data chunk store
- */
-public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor
- *
- * @param dataChunk data chunk
- * @param invertedIndex inverted index
- * @param invertedIndexReverse reverse inverted index
- * @param numberOfRows number of rows
- * @param columnValueSize size of each column value
- */
- public FixedLengthDimensionDataChunk(byte[] dataChunk, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows, int columnValueSize) {
- long totalSize = null != invertedIndex ?
- dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) :
- dataChunk.length;
- dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize,
- DimensionStoreType.FIXEDLENGTH);
- dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param index row id of the chunk
- * @param keyStructureInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int index,
- KeyStructureInfo keyStructureInfo) {
- dataChunkStore.fillRow(index, data, offset);
- return dataChunkStore.getColumnValueSize();
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo) {
- row[columnIndex] = dataChunkStore.getSurrogate(rowId);
- return columnIndex + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = columnVectorInfo.size + offset;
- CarbonColumnVector vector = columnVectorInfo.vector;
- for (int j = offset; j < len; j++) {
- int dict = dataChunkStore.getSurrogate(j);
- if (columnVectorInfo.directDictionaryGenerator == null) {
- vector.putInt(vectorOffset++, dict);
- } else {
- Object valueFromSurrogate =
- columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
- if (valueFromSurrogate == null) {
- vector.putNull(vectorOffset++);
- } else {
- DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
- if (dataType == DataTypes.INT) {
- vector.putInt(vectorOffset++, (int) valueFromSurrogate);
- } else if (dataType == DataTypes.LONG) {
- vector.putLong(vectorOffset++, (long) valueFromSurrogate);
- } else {
- throw new IllegalArgumentException("unsupported data type: " +
- columnVectorInfo.directDictionaryGenerator.getReturnType());
- }
- }
- }
- }
- return column + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = columnVectorInfo.size + offset;
- CarbonColumnVector vector = columnVectorInfo.vector;
- for (int j = offset; j < len; j++) {
- int dict = dataChunkStore.getSurrogate(rowMapping[j]);
- if (columnVectorInfo.directDictionaryGenerator == null) {
- vector.putInt(vectorOffset++, dict);
- } else {
- Object valueFromSurrogate =
- columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
- if (valueFromSurrogate == null) {
- vector.putNull(vectorOffset++);
- } else {
- DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
- if (dataType == DataTypes.INT) {
- vector.putInt(vectorOffset++, (int) valueFromSurrogate);
- } else if (dataType == DataTypes.LONG) {
- vector.putLong(vectorOffset++, (long) valueFromSurrogate);
- } else {
- throw new IllegalArgumentException("unsupported data type: " +
- columnVectorInfo.directDictionaryGenerator.getReturnType());
- }
- }
- }
- }
- return column + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
index fa0777b..5e8618b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/MeasureRawColumnChunk.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -28,7 +28,7 @@ import org.apache.carbondata.core.memory.MemoryException;
/**
* Contains raw measure data
* 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
- * 2. The raw data can be converted to processed chunk using convertToColumnPage method
+ * 2. The raw data can be converted to processed chunk using decodeColumnPage method
* by specifying page number.
*/
public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
@@ -37,7 +37,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
private MeasureColumnChunkReader chunkReader;
- private FileHolder fileReader;
+ private FileReader fileReader;
public MeasureRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
MeasureColumnChunkReader chunkReader) {
@@ -48,14 +48,14 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
/**
* Convert all raw data with all pages to processed ColumnPage
*/
- public ColumnPage[] convertToColumnPage() {
+ public ColumnPage[] decodeAllColumnPages() {
if (columnPages == null) {
columnPages = new ColumnPage[pagesCount];
}
for (int i = 0; i < pagesCount; i++) {
try {
if (columnPages[i] == null) {
- columnPages[i] = chunkReader.convertToColumnPage(this, i);
+ columnPages[i] = chunkReader.decodeColumnPage(this, i);
}
} catch (Exception e) {
throw new RuntimeException(e);
@@ -68,21 +68,21 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
/**
* Convert raw data with specified `columnIndex` processed to ColumnPage
*/
- public ColumnPage convertToColumnPage(int columnIndex) {
- assert columnIndex < pagesCount;
+ public ColumnPage decodeColumnPage(int pageNumber) {
+ assert pageNumber < pagesCount;
if (columnPages == null) {
columnPages = new ColumnPage[pagesCount];
}
try {
- if (columnPages[columnIndex] == null) {
- columnPages[columnIndex] = chunkReader.convertToColumnPage(this, columnIndex);
+ if (columnPages[pageNumber] == null) {
+ columnPages[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);
}
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
- return columnPages[columnIndex];
+ return columnPages[pageNumber];
}
/**
@@ -95,7 +95,7 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
assert index < pagesCount;
try {
- return chunkReader.convertToColumnPage(this, index);
+ return chunkReader.decodeColumnPage(this, index);
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
@@ -111,11 +111,11 @@ public class MeasureRawColumnChunk extends AbstractRawColumnChunk {
}
}
- public void setFileReader(FileHolder fileReader) {
+ public void setFileReader(FileReader fileReader) {
this.fileReader = fileReader;
}
- public FileHolder getFileReader() {
+ public FileReader getFileReader() {
return fileReader;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
new file mode 100644
index 0000000..d03b2de
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionColumnPage.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to variable length dimension data chunk store
+ */
+public class VariableLengthDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor for this class
+ * @param dataChunks
+ * @param invertedIndex
+ * @param invertedIndexReverse
+ * @param numberOfRows
+ */
+ public VariableLengthDimensionColumnPage(byte[] dataChunks, int[] invertedIndex,
+ int[] invertedIndexReverse, int numberOfRows) {
+ long totalSize = null != invertedIndex ?
+ (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + (
+ numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) :
+ (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize,
+ DimensionStoreType.VARIABLELENGTH);
+ dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo restructuringInfo) {
+ // no required in this case because this column chunk is not the part if
+ // mdkey
+ return 0;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo) {
+ return chunkIndex + 1;
+ }
+
+ /**
+ * @return whether column is dictionary column or not
+ */
+ @Override public boolean isNoDicitionaryColumn() {
+ return true;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int i = offset; i < len; i++) {
+ // Considering only String case now as we support only
+ // string in no dictionary case at present.
+ dataChunkStore.fillRow(i, vector, vectorOffset++);
+ }
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ for (int i = offset; i < len; i++) {
+ // Considering only String case now as we support only
+ // string in no dictionary case at present.
+ dataChunkStore.fillRow(filteredRowId[i], vector, vectorOffset++);
+ }
+ return chunkIndex + 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
deleted file mode 100644
index 6c47bf5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to variable length dimension data chunk store
- */
-public class VariableLengthDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor for this class
- * @param dataChunks
- * @param invertedIndex
- * @param invertedIndexReverse
- * @param numberOfRows
- */
- public VariableLengthDimensionDataChunk(byte[] dataChunks, int[] invertedIndex,
- int[] invertedIndexReverse, int numberOfRows) {
- long totalSize = null != invertedIndex ?
- (dataChunks.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) + (
- numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE)) :
- (dataChunks.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
- dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(0, null != invertedIndex, numberOfRows, totalSize,
- DimensionStoreType.VARIABLELENGTH);
- dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunks);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param index row id of the chunk
- * @param restructuringInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int index,
- KeyStructureInfo restructuringInfo) {
- // no required in this case because this column chunk is not the part if
- // mdkey
- return 0;
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo) {
- return columnIndex + 1;
- }
-
- /**
- * @return whether column is dictionary column or not
- */
- @Override public boolean isNoDicitionaryColumn() {
- return true;
- }
-
- /**
- * @return length of each column
- */
- @Override public int getColumnValueSize() {
- return -1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int i = offset; i < len; i++) {
- // Considering only String case now as we support only
- // string in no dictionary case at present.
- dataChunkStore.fillRow(i, vector, vectorOffset++);
- }
- return column + 1;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- CarbonColumnVector vector = columnVectorInfo.vector;
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- for (int i = offset; i < len; i++) {
- // Considering only String case now as we support only
- // string in no dictionary case at present.
- dataChunkStore.fillRow(rowMapping[i], vector, vectorOffset++);
- }
- return column + 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 7b5b9c8..fd81973 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -18,8 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.memory.MemoryException;
@@ -34,20 +34,20 @@ public interface DimensionColumnChunkReader {
* Below method will be used to read the chunk based on block indexes
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocklets to be read
+ * @param columnIndexRange blocklets to be read
* @return dimension column chunks
*/
- DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader, int[][] blockletIndexes)
+ DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Below method will be used to read the chunk based on block index
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndex block to be read
+ * @param columnIndex column to be read
* @return dimension column chunk
*/
- DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int blockletIndex)
+ DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
@@ -58,6 +58,6 @@ public interface DimensionColumnChunkReader {
* @return
* @throws IOException
*/
- DimensionColumnDataChunk convertToDimensionChunk(DimensionRawColumnChunk dimensionRawColumnChunk,
+ DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk dimensionRawColumnChunk,
int pageNumber) throws IOException, MemoryException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 02dc6a2..bf76025 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -18,7 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
@@ -32,20 +32,20 @@ public interface MeasureColumnChunkReader {
* Method to read the blocks data based on block indexes
*
* @param fileReader file reader to read the blocks
- * @param blockIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return measure data chunks
*/
- MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Method to read the blocks data based on block index
*
* @param fileReader file reader to read the blocks
- * @param blockIndex block to be read
+ * @param columnIndex block to be read
* @return measure data chunk
*/
- MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int blockIndex)
+ MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
@@ -55,7 +55,7 @@ public interface MeasureColumnChunkReader {
* @return
* @throws IOException
*/
- ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
index f083612..60950c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReaderV2V3Format.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.format.Encoding;
@@ -55,43 +55,43 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader
* For last column read is separately and process
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocks range to be read
+ * @param columnIndexRange column index range to be read
* @return dimension column chunks
*/
- @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileHolder fileReader,
- final int[][] blockletIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readRawDimensionChunks(final FileReader fileReader,
+ final int[][] columnIndexRange) throws IOException {
// read the column chunk based on block index and add
DimensionRawColumnChunk[] dataChunks =
new DimensionRawColumnChunk[dimensionChunksOffset.size()];
// if blocklet index is empty then return empry data chunk
- if (blockletIndexes.length == 0) {
+ if (columnIndexRange.length == 0) {
return dataChunks;
}
DimensionRawColumnChunk[] groupChunk = null;
int index = 0;
// iterate till block indexes -1 as block index will be in sorted order, so to avoid
// the last column reading in group
- for (int i = 0; i < blockletIndexes.length - 1; i++) {
+ for (int i = 0; i < columnIndexRange.length - 1; i++) {
index = 0;
groupChunk =
- readRawDimensionChunksInGroup(fileReader, blockletIndexes[i][0], blockletIndexes[i][1]);
- for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) {
+ readRawDimensionChunksInGroup(fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
// check last index is present in block index, if it is present then read separately
- if (blockletIndexes[blockletIndexes.length - 1][0] == dimensionChunksOffset.size() - 1) {
- dataChunks[blockletIndexes[blockletIndexes.length - 1][0]] =
- readRawDimensionChunk(fileReader, blockletIndexes[blockletIndexes.length - 1][0]);
+ if (columnIndexRange[columnIndexRange.length - 1][0] == dimensionChunksOffset.size() - 1) {
+ dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
+ readRawDimensionChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]);
}
// otherwise read the data in group
else {
- groupChunk =
- readRawDimensionChunksInGroup(fileReader, blockletIndexes[blockletIndexes.length - 1][0],
- blockletIndexes[blockletIndexes.length - 1][1]);
+ groupChunk = readRawDimensionChunksInGroup(
+ fileReader, columnIndexRange[columnIndexRange.length - 1][0],
+ columnIndexRange[columnIndexRange.length - 1][1]);
index = 0;
- for (int j = blockletIndexes[blockletIndexes.length - 1][0];
- j <= blockletIndexes[blockletIndexes.length - 1][1]; j++) {
+ for (int j = columnIndexRange[columnIndexRange.length - 1][0];
+ j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
@@ -109,7 +109,7 @@ public abstract class AbstractChunkReaderV2V3Format extends AbstractChunkReader
* @return measure raw chunkArray
* @throws IOException
*/
- protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected abstract DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 27a4d89..0dc1c1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -20,12 +20,12 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -61,14 +61,14 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* Below method will be used to read the raw chunk based on block indexes
*
* @param fileReader file reader to read the blocks from file
- * @param blockletIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return dimension column chunks
*/
- @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader,
- int[][] blockletIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionColumnChunk.size()];
- for (int i = 0; i < blockletIndexes.length; i++) {
- for (int j = blockletIndexes[i][0]; j <= blockletIndexes[i][1]; j++) {
+ for (int i = 0; i < columnIndexRange.length; i++) {
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = readRawDimensionChunk(fileReader, j);
}
}
@@ -82,7 +82,7 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
* @param columnIndex column to be read
* @return dimension column chunk
*/
- @Override public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ @Override public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
int columnIndex) throws IOException {
DataChunk dataChunk = dimensionColumnChunk.get(columnIndex);
ByteBuffer buffer = null;
@@ -92,20 +92,20 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
}
DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(columnIndex, buffer, 0,
dataChunk.getDataPageLength(), this);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(1);
rawColumnChunk.setRowCount(new int[] { numberOfRows });
return rawColumnChunk;
}
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException {
int blockIndex = dimensionRawColumnChunk.getColumnIndex();
byte[] dataPage = null;
int[] invertedIndexes = null;
int[] invertedIndexesReverse = null;
int[] rlePage = null;
- FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
+ FileReader fileReader = dimensionRawColumnChunk.getFileReader();
ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(),
@@ -145,23 +145,23 @@ public class CompressedDimensionChunkFileBasedReaderV1 extends AbstractChunkRead
rlePage = null;
}
// fill chunk attributes
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
if (dataChunk.isRowMajor()) {
// to store fixed length column chunk values
- columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex],
- numberOfRows);
+ columnDataChunk = new ColumnGroupDimensionColumnPage(
+ dataPage, eachColumnValueSize[blockIndex], numberOfRows);
}
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
else if (!CarbonUtil
.hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows);
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows, eachColumnValueSize[blockIndex]);
}
return columnDataChunk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index b43f89c..31fa819 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -19,12 +19,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -56,7 +56,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @param columnIndex column to be read
* @return dimension column chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader, int columnIndex)
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException {
int length = 0;
if (dimensionChunksOffset.size() - 1 == columnIndex) {
@@ -73,7 +73,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
}
DimensionRawColumnChunk rawColumnChunk =
new DimensionRawColumnChunk(columnIndex, buffer, 0, length, this);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(1);
rawColumnChunk.setRowCount(new int[] { numberOfRows });
return rawColumnChunk;
@@ -90,7 +90,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
* @return measure raw chunkArray
* @throws IOException
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
long currentDimensionOffset = dimensionChunksOffset.get(startColumnBlockletIndex);
ByteBuffer buffer = null;
@@ -106,7 +106,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
int currentLength = (int) (dimensionChunksOffset.get(i + 1) - dimensionChunksOffset.get(i));
dataChunks[index] =
new DimensionRawColumnChunk(i, buffer, runningLength, currentLength, this);
- dataChunks[index].setFileHolder(fileReader);
+ dataChunks[index].setFileReader(fileReader);
dataChunks[index].setPagesCount(1);
dataChunks[index].setRowCount(new int[] { numberOfRows });
runningLength += currentLength;
@@ -115,7 +115,7 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
return dataChunks;
}
- public DimensionColumnDataChunk convertToDimensionChunk(
+ public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException {
byte[] dataPage = null;
int[] invertedIndexes = null;
@@ -169,23 +169,23 @@ public class CompressedDimensionChunkFileBasedReaderV2 extends AbstractChunkRead
dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]);
}
// fill chunk attributes
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
if (dimensionColumnChunk.isRowMajor()) {
// to store fixed length column chunk values
- columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, eachColumnValueSize[blockIndex],
- numberOfRows);
+ columnDataChunk = new ColumnGroupDimensionColumnPage(
+ dataPage, eachColumnValueSize[blockIndex], numberOfRows);
}
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
else if (!hasEncoding(dimensionColumnChunk.encoders, Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows);
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
numberOfRows, eachColumnValueSize[blockIndex]);
}
return columnDataChunk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
index 1edfd09..60f0b67 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimChunkFileBasedPageLevelReaderV3.java
@@ -20,8 +20,8 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
@@ -70,7 +70,8 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param blockletColumnIndex blocklet index of the column in carbon data file
* @return dimension raw chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
+ @Override
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
int blockletColumnIndex) throws IOException {
// get the current dimension offset
long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
@@ -116,7 +117,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param endBlockletColumnIndex blocklet index of the last dimension column
* @ DimensionRawColumnChunk array
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
// create raw chunk for each dimension column
DimensionRawColumnChunk[] dimensionDataChunks =
@@ -136,7 +137,7 @@ public class CompressedDimChunkFileBasedPageLevelReaderV3
* @param pageNumber number
* @return DimensionColumnDataChunk
*/
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber)
throws IOException, MemoryException {
// data chunk of page
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index 566e9b7..0fdc515 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -20,11 +20,11 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
@@ -78,23 +78,23 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
* 5. Create the raw chunk object and fill the details
*
* @param fileReader reader for reading the column from carbon data file
- * @param blockletColumnIndex blocklet index of the column in carbon data file
+ * @param columnIndex blocklet index of the column in carbon data file
* @return dimension raw chunk
*/
- public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
- int blockletColumnIndex) throws IOException {
+ public DimensionRawColumnChunk readRawDimensionChunk(FileReader fileReader,
+ int columnIndex) throws IOException {
// get the current dimension offset
- long currentDimensionOffset = dimensionChunksOffset.get(blockletColumnIndex);
+ long currentDimensionOffset = dimensionChunksOffset.get(columnIndex);
int length = 0;
// to calculate the length of the data to be read
// column other than last column we can subtract the offset of current column with
// next column and get the total length.
// but for last column we need to use lastDimensionOffset which is the end position
// of the last dimension, we can subtract current dimension offset from lastDimesionOffset
- if (dimensionChunksOffset.size() - 1 == blockletColumnIndex) {
+ if (dimensionChunksOffset.size() - 1 == columnIndex) {
length = (int) (lastDimensionOffsets - currentDimensionOffset);
} else {
- length = (int) (dimensionChunksOffset.get(blockletColumnIndex + 1) - currentDimensionOffset);
+ length = (int) (dimensionChunksOffset.get(columnIndex + 1) - currentDimensionOffset);
}
ByteBuffer buffer = null;
// read the data from carbon data file
@@ -103,15 +103,15 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
}
// get the data chunk which will have all the details about the data pages
DataChunk3 dataChunk = CarbonUtil.readDataChunk3(buffer, 0, length);
- return getDimensionRawColumnChunk(fileReader, blockletColumnIndex, 0, length, buffer,
+ return getDimensionRawColumnChunk(fileReader, columnIndex, 0, length, buffer,
dataChunk);
}
- protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileHolder fileReader,
- int blockletColumnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
+ protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileReader fileReader,
+ int columnIndex, long offset, int length, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
DimensionRawColumnChunk rawColumnChunk =
- new DimensionRawColumnChunk(blockletColumnIndex, buffer, offset, length, this);
+ new DimensionRawColumnChunk(columnIndex, buffer, offset, length, this);
int numberOfPages = dataChunk.getPage_length().size();
byte[][] maxValueOfEachPage = new byte[numberOfPages][];
byte[][] minValueOfEachPage = new byte[numberOfPages][];
@@ -124,7 +124,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
eachPageLength[i] = dataChunk.getData_chunk_list().get(i).getNumberOfRowsInpage();
}
rawColumnChunk.setDataChunkV3(dataChunk);
- rawColumnChunk.setFileHolder(fileReader);
+ rawColumnChunk.setFileReader(fileReader);
rawColumnChunk.setPagesCount(dataChunk.getPage_length().size());
rawColumnChunk.setMaxValues(maxValueOfEachPage);
rawColumnChunk.setMinValues(minValueOfEachPage);
@@ -153,7 +153,7 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
* blocklet index of the last dimension column
* @ DimensionRawColumnChunk array
*/
- protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileHolder fileReader,
+ protected DimensionRawColumnChunk[] readRawDimensionChunksInGroup(FileReader fileReader,
int startBlockletColumnIndex, int endBlockletColumnIndex) throws IOException {
// to calculate the length of the data to be read
// column we can subtract the offset of start column offset with
@@ -188,9 +188,9 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
*
* @param rawColumnPage dimension raw chunk
* @param pageNumber number
- * @return DimensionColumnDataChunk
+ * @return DimensionColumnPage
*/
- @Override public DimensionColumnDataChunk convertToDimensionChunk(
+ @Override public DimensionColumnPage decodeColumnPage(
DimensionRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException {
// data chunk of blocklet column
DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
@@ -228,20 +228,19 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
return false;
}
- protected DimensionColumnDataChunk decodeDimension(DimensionRawColumnChunk rawColumnPage,
+ protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset)
throws IOException, MemoryException {
if (isEncodedWithMeta(pageMetadata)) {
ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, offset);
- return new ColumnPageWrapper(decodedPage,
- eachColumnValueSize[rawColumnPage.getColumnIndex()]);
+ return new ColumnPageWrapper(decodedPage);
} else {
// following code is for backward compatibility
return decodeDimensionLegacy(rawColumnPage, pageData, pageMetadata, offset);
}
}
- private DimensionColumnDataChunk decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage,
+ private DimensionColumnPage decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage,
ByteBuffer pageData, DataChunk2 pageMetadata, int offset) {
byte[] dataPage;
int[] rlePage;
@@ -267,18 +266,18 @@ public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkRead
eachColumnValueSize[rawColumnPage.getColumnIndex()]);
}
- DimensionColumnDataChunk columnDataChunk = null;
+ DimensionColumnPage columnDataChunk = null;
// if no dictionary column then first create a no dictionary column chunk
// and set to data chunk instance
if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) {
columnDataChunk =
- new VariableLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
pageMetadata.getNumberOfRowsInpage());
} else {
// to store fixed length column chunk values
columnDataChunk =
- new FixedLengthDimensionDataChunk(dataPage, invertedIndexes, invertedIndexesReverse,
+ new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse,
pageMetadata.getNumberOfRowsInpage(),
eachColumnValueSize[rawColumnPage.getColumnIndex()]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index 2239a2b..a3ed339 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -56,36 +56,40 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
* separately and process
*
* @param fileReader file reader to read the blocks from file
- * @param blockIndexes blocks range to be read
+ * @param columnIndexRange blocks range to be read, columnIndexGroup[i] is one group, inside the
+ * group, columnIndexGroup[i][0] is start column index,
+ * and columnIndexGroup[i][1] is end column index
* @return measure column chunks
* @throws IOException
*/
- public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
+ public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// read the column chunk based on block index and add
MeasureRawColumnChunk[] dataChunks =
new MeasureRawColumnChunk[measureColumnChunkOffsets.size()];
- if (blockIndexes.length == 0) {
+ if (columnIndexRange.length == 0) {
return dataChunks;
}
MeasureRawColumnChunk[] groupChunk = null;
int index = 0;
- for (int i = 0; i < blockIndexes.length - 1; i++) {
+ for (int i = 0; i < columnIndexRange.length - 1; i++) {
index = 0;
- groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[i][0], blockIndexes[i][1]);
- for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[i][0], columnIndexRange[i][1]);
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
- if (blockIndexes[blockIndexes.length - 1][0] == measureColumnChunkOffsets.size() - 1) {
- dataChunks[blockIndexes[blockIndexes.length - 1][0]] =
- readRawMeasureChunk(fileReader, blockIndexes[blockIndexes.length - 1][0]);
+ if (columnIndexRange[columnIndexRange.length - 1][0] == measureColumnChunkOffsets.size() - 1) {
+ dataChunks[columnIndexRange[columnIndexRange.length - 1][0]] =
+ readRawMeasureChunk(fileReader, columnIndexRange[columnIndexRange.length - 1][0]);
} else {
- groupChunk = readRawMeasureChunksInGroup(fileReader, blockIndexes[blockIndexes.length - 1][0],
- blockIndexes[blockIndexes.length - 1][1]);
+ groupChunk = readRawMeasureChunksInGroup(
+ fileReader, columnIndexRange[columnIndexRange.length - 1][0],
+ columnIndexRange[columnIndexRange.length - 1][1]);
index = 0;
- for (int j = blockIndexes[blockIndexes.length - 1][0];
- j <= blockIndexes[blockIndexes.length - 1][1]; j++) {
+ for (int j = columnIndexRange[columnIndexRange.length - 1][0];
+ j <= columnIndexRange[columnIndexRange.length - 1][1]; j++) {
dataChunks[j] = groupChunk[index++];
}
}
@@ -112,12 +116,12 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
* data from
*
* @param fileReader file reader to read the data
- * @param startColumnBlockletIndex first column blocklet index to be read
- * @param endColumnBlockletIndex end column blocklet index to be read
+ * @param startColumnIndex first column index to be read
+ * @param endColumnIndex end column index to be read
* @return measure raw chunkArray
* @throws IOException
*/
- protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
+ protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index ae55375..f0c1b75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -56,14 +56,14 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* Method to read the blocks data based on block indexes
*
* @param fileReader file reader to read the blocks
- * @param blockIndexes blocks to be read
+ * @param columnIndexRange blocks to be read
* @return measure data chunks
*/
- @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readRawMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
MeasureRawColumnChunk[] datChunk = new MeasureRawColumnChunk[measureColumnChunks.size()];
- for (int i = 0; i < blockIndexes.length; i++) {
- for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
+ for (int i = 0; i < columnIndexRange.length; i++) {
+ for (int j = columnIndexRange[i][0]; j <= columnIndexRange[i][1]; j++) {
datChunk[j] = readRawMeasureChunk(fileReader, j);
}
}
@@ -77,7 +77,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
* @param columnIndex column to be read
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex)
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
DataChunk dataChunk = measureColumnChunks.get(columnIndex);
ByteBuffer buffer = fileReader
@@ -91,7 +91,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
}
@Override
- public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
int blockIndex = measureRawColumnChunk.getColumnIndex();
DataChunk dataChunk = measureColumnChunks.get(blockIndex);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index d61f98a..04d6e2e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -48,7 +48,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
}
@Override
- public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader, int columnIndex)
+ public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
int dataLength = 0;
if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
@@ -83,30 +83,30 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
* data from
*
* @param fileReader file reader to read the data
- * @param startColumnBlockletIndex first column blocklet index to be read
- * @param endColumnBlockletIndex end column blocklet index to be read
+ * @param startColumnIndex first column blocklet index to be read
+ * @param endColumnIndex end column blocklet index to be read
* @return measure raw chunkArray
* @throws IOException
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
- long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex);
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException {
+ long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
ByteBuffer buffer = null;
synchronized (fileReader) {
buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset,
- (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset));
+ (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
}
MeasureRawColumnChunk[] dataChunks =
- new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
int runningLength = 0;
int index = 0;
- for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ for (int i = startColumnIndex; i <= endColumnIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
MeasureRawColumnChunk measureRawColumnChunk =
new MeasureRawColumnChunk(i, buffer, runningLength, currentLength, this);
- measureRawColumnChunk.setFileReader(fileReader);
measureRawColumnChunk.setRowCount(new int[] { numberOfRows });
+ measureRawColumnChunk.setFileReader(fileReader);
measureRawColumnChunk.setPagesCount(1);
dataChunks[index] = measureRawColumnChunk;
runningLength += currentLength;
@@ -115,7 +115,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
return dataChunks;
}
- public ColumnPage convertToColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
+ public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException, MemoryException {
int copyPoint = (int) measureRawColumnChunk.getOffSet();
int blockIndex = measureRawColumnChunk.getColumnIndex();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 4f6987b..6dc02a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -70,7 +70,7 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
* @param columnIndex column to be read
* @return measure raw chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader,
int columnIndex) throws IOException {
int dataLength = 0;
// to calculate the length of the data to be read
@@ -99,9 +99,8 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
dataChunk);
}
- protected MeasureRawColumnChunk getMeasureRawColumnChunk(FileHolder fileReader,
- int columnIndex, long offset, int dataLength, ByteBuffer buffer,
- DataChunk3 dataChunk) {
+ MeasureRawColumnChunk getMeasureRawColumnChunk(FileReader fileReader, int columnIndex,
+ long offset, int dataLength, ByteBuffer buffer, DataChunk3 dataChunk) {
// creating a raw chunks instance and filling all the details
MeasureRawColumnChunk rawColumnChunk =
new MeasureRawColumnChunk(columnIndex, buffer, offset, dataLength, this);
@@ -140,30 +139,30 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
*
* @param fileReader
* reader which will be used to read the measure columns data from file
- * @param startColumnBlockletIndex
- * blocklet index of the first measure column
- * @param endColumnBlockletIndex
- * blocklet index of the last measure column
+ * @param startColumnIndex
+ * column index of the first measure column
+ * @param endColumnIndex
+ * column index of the last measure column
* @return MeasureRawColumnChunk array
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
- int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
+ int startColumnIndex, int endColumnIndex) throws IOException {
// to calculate the length of the data to be read
// column we can subtract the offset of start column offset with
// end column+1 offset and get the total length.
- long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnBlockletIndex);
+ long currentMeasureOffset = measureColumnChunkOffsets.get(startColumnIndex);
ByteBuffer buffer = null;
// read the data from carbon data file
synchronized (fileReader) {
buffer = fileReader.readByteBuffer(filePath, currentMeasureOffset,
- (int) (measureColumnChunkOffsets.get(endColumnBlockletIndex + 1) - currentMeasureOffset));
+ (int) (measureColumnChunkOffsets.get(endColumnIndex + 1) - currentMeasureOffset));
}
// create raw chunk for each measure column
MeasureRawColumnChunk[] measureDataChunk =
- new MeasureRawColumnChunk[endColumnBlockletIndex - startColumnBlockletIndex + 1];
+ new MeasureRawColumnChunk[endColumnIndex - startColumnIndex + 1];
int runningLength = 0;
int index = 0;
- for (int i = startColumnBlockletIndex; i <= endColumnBlockletIndex; i++) {
+ for (int i = startColumnIndex; i <= endColumnIndex; i++) {
int currentLength =
(int) (measureColumnChunkOffsets.get(i + 1) - measureColumnChunkOffsets.get(i));
DataChunk3 dataChunk =
@@ -180,25 +179,25 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
/**
* Below method will be used to convert the compressed measure chunk raw data to actual data
*
- * @param rawColumnPage measure raw chunk
+ * @param rawColumnChunk measure raw chunk
* @param pageNumber number
- * @return DimensionColumnDataChunk
+ * @return DimensionColumnPage
*/
@Override
- public ColumnPage convertToColumnPage(
- MeasureRawColumnChunk rawColumnPage, int pageNumber)
+ public ColumnPage decodeColumnPage(
+ MeasureRawColumnChunk rawColumnChunk, int pageNumber)
throws IOException, MemoryException {
// data chunk of blocklet column
- DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
+ DataChunk3 dataChunk3 = rawColumnChunk.getDataChunkV3();
// data chunk of page
DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
// calculating the start point of data
// as buffer can contain multiple column data, start point will be datachunkoffset +
// data chunk length + page offset
- int offset = (int) rawColumnPage.getOffSet() +
- measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) +
+ int offset = (int) rawColumnChunk.getOffSet() +
+ measureColumnChunkLength.get(rawColumnChunk.getColumnIndex()) +
dataChunk3.getPage_offset().get(pageNumber);
- ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnPage.getRawData(), offset);
+ ColumnPage decodedPage = decodeMeasure(pageMetadata, rawColumnChunk.getRawData(), offset);
decodedPage.setNullBits(getNullBitSet(pageMetadata.presence));
return decodedPage;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
index 31ff4c0..6b37575 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMsrChunkFileBasedPageLevelReaderV3.java
@@ -20,7 +20,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
@@ -66,7 +66,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param blockletColumnIndex blocklet index of the column in carbon data file
* @return measure raw chunk
*/
- @Override public MeasureRawColumnChunk readRawMeasureChunk(FileHolder fileReader,
+ @Override public MeasureRawColumnChunk readRawMeasureChunk(FileReader fileReader,
int blockletColumnIndex) throws IOException {
int dataLength = 0;
// to calculate the length of the data to be read
@@ -110,7 +110,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param endColumnBlockletIndex blocklet index of the last measure column
* @return MeasureRawColumnChunk array
*/
- protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
+ protected MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileReader fileReader,
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException {
// create raw chunk for each measure column
MeasureRawColumnChunk[] measureDataChunk =
@@ -130,7 +130,7 @@ public class CompressedMsrChunkFileBasedPageLevelReaderV3
* @param pageNumber number
* @return DimensionColumnDataChunk
*/
- @Override public ColumnPage convertToColumnPage(
+ @Override public ColumnPage decodeColumnPage(
MeasureRawColumnChunk rawColumnPage, int pageNumber)
throws IOException, MemoryException {
// data chunk of blocklet column
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index fbdb499..c89ecc3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -17,48 +17,45 @@
package org.apache.carbondata.core.datastore.chunk.store;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-public class ColumnPageWrapper implements DimensionColumnDataChunk {
+public class ColumnPageWrapper implements DimensionColumnPage {
private ColumnPage columnPage;
- private int columnValueSize;
- public ColumnPageWrapper(ColumnPage columnPage, int columnValueSize) {
+ public ColumnPageWrapper(ColumnPage columnPage) {
this.columnPage = columnPage;
- this.columnValueSize = columnValueSize;
}
@Override
- public int fillChunkData(byte[] data, int offset, int columnIndex,
- KeyStructureInfo restructuringInfo) {
+ public int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
+ public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column,
+ public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
KeyStructureInfo restructuringInfo) {
throw new UnsupportedOperationException("internal error");
}
@Override
- public byte[] getChunkData(int columnIndex) {
- return columnPage.getBytes(columnIndex);
+ public byte[] getChunkData(int rowId) {
+ return columnPage.getBytes(rowId);
}
@Override
@@ -66,7 +63,7 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
throw new UnsupportedOperationException("internal error");
}
- @Override public int getInvertedReverseIndex(int invertedIndex) {
+ @Override public int getInvertedReverseIndex(int rowId) {
throw new UnsupportedOperationException("internal error");
}
@@ -76,17 +73,12 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
}
@Override
- public int getColumnValueSize() {
- return columnValueSize;
- }
-
- @Override
public boolean isExplicitSorted() {
return false;
}
@Override
- public int compareTo(int index, byte[] compareValue) {
+ public int compareTo(int rowId, byte[] compareValue) {
throw new UnsupportedOperationException("internal error");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
index 5072c75..28aed5b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
@@ -69,10 +69,10 @@ public interface DimensionDataChunkStore {
/**
* Below method will be used to get the reverse Inverted Index
- * @param invertedIndex
+ * @param rowId
* @return reverse Inverted Index
*/
- int getInvertedReverseIndex(int invertedIndex);
+ int getInvertedReverseIndex(int rowId);
/**
* Below method will be used to get the surrogate key of the
@@ -102,9 +102,9 @@ public interface DimensionDataChunkStore {
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- int compareTo(int index, byte[] compareValue);
+ int compareTo(int rowId, byte[] compareValue);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
index 8656878..41218d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
@@ -97,13 +97,13 @@ public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimension
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
return ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0,
+ .compareTo(data, rowId * columnValueSize, columnValueSize, compareValue, 0,
columnValueSize);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index db83198..f498c6e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -162,7 +162,7 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
}
}
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -171,11 +171,11 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
// length
// get the offset of set of data
- int currentDataOffset = dataOffsets[index];
+ int currentDataOffset = dataOffsets[rowId];
short length = 0;
// calculating the length of data
- if (index < numberOfRows - 1) {
- length = (short) (dataOffsets[index + 1] - (currentDataOffset
+ if (rowId < numberOfRows - 1) {
+ length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
} else {
// for last record
[41/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
deleted file mode 100644
index 90178b1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ /dev/null
@@ -1,971 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cacheable;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.indexstore.BlockMetaInfo;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
-import org.apache.carbondata.core.indexstore.row.DataMapRow;
-import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
-import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.xerial.snappy.Snappy;
-
-/**
- * Datamap implementation for blocklet.
- */
-public class BlockletDataMap extends AbstractCoarseGrainDataMap implements Cacheable {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockletDataMap.class.getName());
-
- private static int KEY_INDEX = 0;
-
- private static int MIN_VALUES_INDEX = 1;
-
- private static int MAX_VALUES_INDEX = 2;
-
- private static int ROW_COUNT_INDEX = 3;
-
- private static int FILE_PATH_INDEX = 4;
-
- private static int PAGE_COUNT_INDEX = 5;
-
- private static int VERSION_INDEX = 6;
-
- private static int SCHEMA_UPADATED_TIME_INDEX = 7;
-
- private static int BLOCK_INFO_INDEX = 8;
-
- private static int BLOCK_FOOTER_OFFSET = 9;
-
- private static int LOCATIONS = 10;
-
- private static int BLOCKLET_ID_INDEX = 11;
-
- private static int BLOCK_LENGTH = 12;
-
- private static int TASK_MIN_VALUES_INDEX = 0;
-
- private static int TASK_MAX_VALUES_INDEX = 1;
-
- private static int SCHEMA = 2;
-
- private static int INDEX_PATH = 3;
-
- private static int INDEX_FILE_NAME = 4;
-
- private static int SEGMENTID = 5;
-
- private UnsafeMemoryDMStore unsafeMemoryDMStore;
-
- private UnsafeMemoryDMStore unsafeMemorySummaryDMStore;
-
- private SegmentProperties segmentProperties;
-
- private int[] columnCardinality;
-
- @Override
- public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
- long startTime = System.currentTimeMillis();
- assert (dataMapModel instanceof BlockletDataMapModel);
- BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- List<DataFileFooter> indexInfo = fileFooterConverter
- .getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
- Path path = new Path(blockletDataMapInfo.getFilePath());
- byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- byte[] segmentId =
- blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- DataMapRowImpl summaryRow = null;
- byte[] schemaBinary = null;
- // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
- // is id assigned to a blocklet within a part file
- String tempFilePath = null;
- int relativeBlockletId = 0;
- for (DataFileFooter fileFooter : indexInfo) {
- if (segmentProperties == null) {
- List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
- schemaBinary = convertSchemaToBinary(columnInTable);
- columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
- segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
- createSchema(segmentProperties);
- createSummarySchema(segmentProperties, schemaBinary, filePath, fileName,
- segmentId);
- }
- TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
- BlockMetaInfo blockMetaInfo =
- blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
- // Here it loads info about all blocklets of index
- // Only add if the file exists physically. There are scenarios which index file exists inside
- // merge index but related carbondata files are deleted. In that case we first check whether
- // the file exists physically or not
- if (blockMetaInfo != null) {
- if (fileFooter.getBlockletList() == null) {
- // This is old store scenario, here blocklet information is not available in index file so
- // load only block info
- summaryRow =
- loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- blockMetaInfo);
- } else {
- // blocklet ID will start from 0 again only when part file path is changed
- if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
- tempFilePath = blockInfo.getFilePath();
- relativeBlockletId = 0;
- }
- summaryRow =
- loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- blockMetaInfo, relativeBlockletId);
- // this is done because relative blocklet id need to be incremented based on the
- // total number of blocklets
- relativeBlockletId += fileFooter.getBlockletList().size();
- }
- }
- }
- if (unsafeMemoryDMStore != null) {
- unsafeMemoryDMStore.finishWriting();
- }
- if (null != unsafeMemorySummaryDMStore) {
- addTaskSummaryRowToUnsafeMemoryStore(
- summaryRow,
- schemaBinary,
- filePath,
- fileName,
- segmentId);
- unsafeMemorySummaryDMStore.finishWriting();
- }
- LOGGER.info(
- "Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (
- System.currentTimeMillis() - startTime));
- }
-
- private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
- SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- List<BlockletInfo> blockletList = fileFooter.getBlockletList();
- CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
- // Add one row to maintain task level min max for segment pruning
- if (!blockletList.isEmpty() && summaryRow == null) {
- summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
- }
- for (int index = 0; index < blockletList.size(); index++) {
- DataMapRow row = new DataMapRowImpl(schema);
- int ordinal = 0;
- int taskMinMaxOrdinal = 0;
- BlockletInfo blockletInfo = blockletList.get(index);
-
- // add start key as index key
- row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
-
- BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
- byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
- // compute and set task level min values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues,
- TASK_MIN_VALUES_INDEX, true);
- ordinal++;
- taskMinMaxOrdinal++;
- byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
- // compute and set task level max values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
- TASK_MAX_VALUES_INDEX, false);
- ordinal++;
-
- row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
-
- // add file path
- byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- row.setByteArray(filePathBytes, ordinal++);
-
- // add pages
- row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
-
- // add version number
- row.setShort(fileFooter.getVersionId().number(), ordinal++);
-
- // add schema updated time
- row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
-
- // add blocklet info
- byte[] serializedData;
- try {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- DataOutput dataOutput = new DataOutputStream(stream);
- blockletInfo.write(dataOutput);
- serializedData = stream.toByteArray();
- row.setByteArray(serializedData, ordinal++);
- // Add block footer offset, it is used if we need to read footer of block
- row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
- setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
- ordinal++;
- // for relative blockelt id i.e blocklet id that belongs to a particular part file
- row.setShort((short) relativeBlockletId++, ordinal++);
- // Store block size
- row.setLong(blockMetaInfo.getSize(), ordinal);
- unsafeMemoryDMStore.addIndexRowToUnsafe(row);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- return summaryRow;
- }
-
- private void setLocations(String[] locations, DataMapRow row, int ordinal)
- throws UnsupportedEncodingException {
- // Add location info
- String locationStr = StringUtils.join(locations, ',');
- row.setByteArray(locationStr.getBytes(CarbonCommonConstants.DEFAULT_CHARSET), ordinal);
- }
-
- /**
- * Load information for the block.It is the case can happen only for old stores
- * where blocklet information is not available in index file. So load only block information
- * and read blocklet information in executor.
- */
- private DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter,
- SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- BlockMetaInfo blockMetaInfo) {
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
- CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
- // Add one row to maintain task level min max for segment pruning
- if (summaryRow == null) {
- summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
- }
- DataMapRow row = new DataMapRowImpl(schema);
- int ordinal = 0;
- int taskMinMaxOrdinal = 0;
- // add start key as index key
- row.setByteArray(blockletIndex.getBtreeIndex().getStartKey(), ordinal++);
-
- BlockletMinMaxIndex minMaxIndex = blockletIndex.getMinMaxIndex();
- byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
- byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
- // update min max values in case of old store
- byte[][] updatedMinValues =
- CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
- byte[][] updatedMaxValues =
- CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMinValues), ordinal);
- // compute and set task level min values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMinValues,
- TASK_MIN_VALUES_INDEX, true);
- ordinal++;
- taskMinMaxOrdinal++;
- row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMaxValues), ordinal);
- // compute and set task level max values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMaxValues,
- TASK_MAX_VALUES_INDEX, false);
- ordinal++;
-
- row.setInt((int)fileFooter.getNumberOfRows(), ordinal++);
-
- // add file path
- byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- row.setByteArray(filePathBytes, ordinal++);
-
- // add pages
- row.setShort((short) 0, ordinal++);
-
- // add version number
- row.setShort(fileFooter.getVersionId().number(), ordinal++);
-
- // add schema updated time
- row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
-
- // add blocklet info
- row.setByteArray(new byte[0], ordinal++);
-
- row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
- try {
- setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
- ordinal++;
- // for relative blocklet id. Value is -1 because in case of old store blocklet info will
- // not be present in the index file and in that case we will not knwo the total number of
- // blocklets
- row.setShort((short) -1, ordinal++);
-
- // store block size
- row.setLong(blockMetaInfo.getSize(), ordinal);
- unsafeMemoryDMStore.addIndexRowToUnsafe(row);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- return summaryRow;
- }
-
- private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] schemaBinary,
- byte[] filePath, byte[] fileName, byte[] segmentId) {
- // write the task summary info to unsafe memory store
- if (null != summaryRow) {
- // Add column schema , it is useful to generate segment properties in executor.
- // So we no need to read footer again there.
- if (schemaBinary != null) {
- summaryRow.setByteArray(schemaBinary, SCHEMA);
- }
- summaryRow.setByteArray(filePath, INDEX_PATH);
- summaryRow.setByteArray(fileName, INDEX_FILE_NAME);
- summaryRow.setByteArray(segmentId, SEGMENTID);
- try {
- unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * Fill the measures min values with minimum , this is needed for backward version compatability
- * as older versions don't store min values for measures
- */
- private byte[][] updateMinValues(byte[][] minValues, int[] minMaxLen) {
- byte[][] updatedValues = minValues;
- if (minValues.length < minMaxLen.length) {
- updatedValues = new byte[minMaxLen.length][];
- System.arraycopy(minValues, 0, updatedValues, 0, minValues.length);
- List<CarbonMeasure> measures = segmentProperties.getMeasures();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- for (int i = 0; i < measures.size(); i++) {
- buffer.rewind();
- DataType dataType = measures.get(i).getDataType();
- if (dataType == DataTypes.BYTE) {
- buffer.putLong(Byte.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.SHORT) {
- buffer.putLong(Short.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.INT) {
- buffer.putLong(Integer.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.LONG) {
- buffer.putLong(Long.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (DataTypes.isDecimal(dataType)) {
- updatedValues[minValues.length + i] =
- DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MIN_VALUE));
- } else {
- buffer.putDouble(Double.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- }
- }
- }
- return updatedValues;
- }
-
- /**
- * Fill the measures max values with maximum , this is needed for backward version compatability
- * as older versions don't store max values for measures
- */
- private byte[][] updateMaxValues(byte[][] maxValues, int[] minMaxLen) {
- byte[][] updatedValues = maxValues;
- if (maxValues.length < minMaxLen.length) {
- updatedValues = new byte[minMaxLen.length][];
- System.arraycopy(maxValues, 0, updatedValues, 0, maxValues.length);
- List<CarbonMeasure> measures = segmentProperties.getMeasures();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- for (int i = 0; i < measures.size(); i++) {
- buffer.rewind();
- DataType dataType = measures.get(i).getDataType();
- if (dataType == DataTypes.BYTE) {
- buffer.putLong(Byte.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.SHORT) {
- buffer.putLong(Short.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.INT) {
- buffer.putLong(Integer.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.LONG) {
- buffer.putLong(Long.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (DataTypes.isDecimal(dataType)) {
- updatedValues[maxValues.length + i] =
- DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MAX_VALUE));
- } else {
- buffer.putDouble(Double.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- }
- }
- }
- return updatedValues;
- }
-
- private DataMapRow addMinMax(int[] minMaxLen, CarbonRowSchema carbonRowSchema,
- byte[][] minValues) {
- CarbonRowSchema[] minSchemas =
- ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
- DataMapRow minRow = new DataMapRowImpl(minSchemas);
- int minOrdinal = 0;
- // min value adding
- for (int i = 0; i < minMaxLen.length; i++) {
- minRow.setByteArray(minValues[i], minOrdinal++);
- }
- return minRow;
- }
-
- /**
- * This method will compute min/max values at task level
- *
- * @param taskMinMaxRow
- * @param minMaxLen
- * @param carbonRowSchema
- * @param minMaxValue
- * @param ordinal
- * @param isMinValueComparison
- */
- private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen,
- CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal,
- boolean isMinValueComparison) {
- DataMapRow row = taskMinMaxRow.getRow(ordinal);
- byte[][] updatedMinMaxValues = minMaxValue;
- if (null == row) {
- CarbonRowSchema[] minSchemas =
- ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
- row = new DataMapRowImpl(minSchemas);
- } else {
- byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
- // Compare and update min max values
- for (int i = 0; i < minMaxLen.length; i++) {
- int compare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
- if (isMinValueComparison) {
- if (compare < 0) {
- updatedMinMaxValues[i] = existingMinMaxValues[i];
- }
- } else if (compare > 0) {
- updatedMinMaxValues[i] = existingMinMaxValues[i];
- }
- }
- }
- int minMaxOrdinal = 0;
- // min/max value adding
- for (int i = 0; i < minMaxLen.length; i++) {
- row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
- }
- taskMinMaxRow.setRow(row, ordinal);
- }
-
- private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
- List<CarbonRowSchema> indexSchemas = new ArrayList<>();
-
- // Index key
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
- getMinMaxSchema(segmentProperties, indexSchemas);
-
- // for number of rows.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
-
- // for table block path
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for number of pages.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for version number.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for schema updated time.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- //for blocklet info
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for block footer offset.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- // for locations
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for storing block length.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- unsafeMemoryDMStore =
- new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
- }
-
- /**
- * Creates the schema to store summary information or the information which can be stored only
- * once per datamap. It stores datamap level max/min of each column and partition information of
- * datamap
- * @param segmentProperties
- * @throws MemoryException
- */
- private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary,
- byte[] filePath, byte[] fileName, byte[] segmentId)
- throws MemoryException {
- List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
- getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
- // for storing column schema
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
- // for storing file path
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
- // for storing file name
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
- // for storing segmentid
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
- unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
- taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
- }
-
- private void getMinMaxSchema(SegmentProperties segmentProperties,
- List<CarbonRowSchema> minMaxSchemas) {
- // Index key
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- // do it 2 times, one for min and one for max.
- for (int k = 0; k < 2; k++) {
- CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
- for (int i = 0; i < minMaxLen.length; i++) {
- if (minMaxLen[i] <= 0) {
- mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
- } else {
- mapSchemas[i] =
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
- }
- }
- CarbonRowSchema mapSchema =
- new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
- mapSchemas);
- minMaxSchemas.add(mapSchema);
- }
- }
-
- @Override
- public boolean isScanRequired(FilterResolverIntf filterExp) {
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
- boolean isScanRequired = FilterExpressionProcessor.isScanRequired(
- filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
- getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
- if (isScanRequired) {
- return true;
- }
- }
- return false;
- }
-
- private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
- if (unsafeMemoryDMStore.getRowCount() == 0) {
- return new ArrayList<>();
- }
- List<Blocklet> blocklets = new ArrayList<>();
- if (filterExp == null) {
- int rowCount = unsafeMemoryDMStore.getRowCount();
- for (int i = 0; i < rowCount; i++) {
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
- blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
- }
- } else {
- // Remove B-tree jump logic as start and end key prepared is not
- // correct for old store scenarios
- int startIndex = 0;
- int endIndex = unsafeMemoryDMStore.getRowCount();
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- while (startIndex < endIndex) {
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
- int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
- String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX),
- CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- boolean isValid =
- addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX),
- getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
- if (isValid) {
- blocklets.add(createBlocklet(safeRow, blockletId));
- }
- startIndex++;
- }
- }
- return blocklets;
- }
-
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions) {
- if (unsafeMemoryDMStore.getRowCount() == 0) {
- return new ArrayList<>();
- }
- // if it has partitioned datamap but there is no partitioned information stored, it means
- // partitions are dropped so return empty list.
- if (partitions != null) {
- // First get the partitions which are stored inside datamap.
- String[] fileDetails = getFileDetails();
- // Check the exact match of partition information inside the stored partitions.
- boolean found = false;
- Path folderPath = new Path(fileDetails[0]);
- for (PartitionSpec spec : partitions) {
- if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) {
- found = true;
- break;
- }
- }
- if (!found) {
- return new ArrayList<>();
- }
- }
- // Prune with filters if the partitions are existed in this datamap
- return prune(filterExp, segmentProperties);
- }
-
- private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
- boolean needToScan = false;
- if (spec.getUuid() != null) {
- String[] split = spec.getUuid().split("_");
- if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil
- .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) {
- needToScan = true;
- }
- } else {
- needToScan = true;
- }
- return needToScan;
- }
-
- /**
- * select the blocks based on column min and max value
- *
- * @param filterExecuter
- * @param maxValue
- * @param minValue
- * @param filePath
- * @param blockletId
- * @return
- */
- private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue,
- byte[][] minValue, String filePath, int blockletId) {
- BitSet bitSet = null;
- if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
- String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
- // this case will come in case of old store where index file does not contain the
- // blocklet information
- if (blockletId != -1) {
- uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
- }
- bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
- .isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
- } else {
- bitSet = filterExecuter.isScanRequired(maxValue, minValue);
- }
- if (!bitSet.isEmpty()) {
- return true;
- } else {
- return false;
- }
- }
-
- public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
- int index = Integer.parseInt(blockletId);
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(index).convertToSafeRow();
- return createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX));
- }
-
- private byte[][] getMinMaxValue(DataMapRow row, int index) {
- DataMapRow minMaxRow = row.getRow(index);
- byte[][] minMax = new byte[minMaxRow.getColumnCount()][];
- for (int i = 0; i < minMax.length; i++) {
- minMax[i] = minMaxRow.getByteArray(i);
- }
- return minMax;
- }
-
- private ExtendedBlocklet createBlocklet(DataMapRow row, int blockletId) {
- ExtendedBlocklet blocklet = new ExtendedBlocklet(
- new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
- blockletId + "");
- BlockletDetailInfo detailInfo = new BlockletDetailInfo();
- detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
- detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
- detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
- detailInfo.setBlockletId((short) blockletId);
- detailInfo.setDimLens(columnCardinality);
- detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
- byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
- BlockletInfo blockletInfo = null;
- try {
- if (byteArray.length > 0) {
- blockletInfo = new BlockletInfo();
- ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
- DataInputStream inputStream = new DataInputStream(stream);
- blockletInfo.readFields(inputStream);
- inputStream.close();
- }
- blocklet.setLocation(
- new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET)
- .split(","));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- detailInfo.setBlockletInfo(blockletInfo);
- blocklet.setDetailInfo(detailInfo);
- detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
- detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
- detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
- return blocklet;
- }
-
- private String[] getFileDetails() {
- try {
- String[] fileDetails = new String[3];
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
- fileDetails[0] =
- new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
- fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME),
- CarbonCommonConstants.DEFAULT_CHARSET);
- fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID),
- CarbonCommonConstants.DEFAULT_CHARSET);
- return fileDetails;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
-
- /**
- * Binary search used to get the first tentative index row based on
- * search key
- *
- * @param key search key
- * @return first tentative block
- */
- private int findStartIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
- int childNodeIndex;
- int low = 0;
- int high = unsafeMemoryDMStore.getRowCount() - 1;
- int mid = 0;
- int compareRes = -1;
- //
- while (low <= high) {
- mid = (low + high) >>> 1;
- // compare the entries
- compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
- if (compareRes < 0) {
- high = mid - 1;
- } else if (compareRes > 0) {
- low = mid + 1;
- } else {
- // if key is matched then get the first entry
- int currentPos = mid;
- while (currentPos - 1 >= 0
- && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos - 1)) == 0) {
- currentPos--;
- }
- mid = currentPos;
- break;
- }
- }
- // if compare result is less than zero then we
- // and mid is more than 0 then we need to previous block as duplicates
- // record can be present
- if (compareRes < 0) {
- if (mid > 0) {
- mid--;
- }
- childNodeIndex = mid;
- } else {
- childNodeIndex = mid;
- }
- // get the leaf child
- return childNodeIndex;
- }
-
- /**
- * Binary search used to get the last tentative block based on
- * search key
- *
- * @param key search key
- * @return first tentative block
- */
- private int findEndIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
- int childNodeIndex;
- int low = 0;
- int high = unsafeMemoryDMStore.getRowCount() - 1;
- int mid = 0;
- int compareRes = -1;
- //
- while (low <= high) {
- mid = (low + high) >>> 1;
- // compare the entries
- compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
- if (compareRes < 0) {
- high = mid - 1;
- } else if (compareRes > 0) {
- low = mid + 1;
- } else {
- int currentPos = mid;
- // if key is matched then get the first entry
- while (currentPos + 1 < unsafeMemoryDMStore.getRowCount()
- && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos + 1)) == 0) {
- currentPos++;
- }
- mid = currentPos;
- break;
- }
- }
- // if compare result is less than zero then we
- // and mid is more than 0 then we need to previous block as duplicates
- // record can be present
- if (compareRes < 0) {
- if (mid > 0) {
- mid--;
- }
- childNodeIndex = mid;
- } else {
- childNodeIndex = mid;
- }
- return childNodeIndex;
- }
-
- private DataMapRow convertToRow(IndexKey key) {
- ByteBuffer buffer =
- ByteBuffer.allocate(key.getDictionaryKeys().length + key.getNoDictionaryKeys().length + 8);
- buffer.putInt(key.getDictionaryKeys().length);
- buffer.putInt(key.getNoDictionaryKeys().length);
- buffer.put(key.getDictionaryKeys());
- buffer.put(key.getNoDictionaryKeys());
- DataMapRowImpl dataMapRow = new DataMapRowImpl(unsafeMemoryDMStore.getSchema());
- dataMapRow.setByteArray(buffer.array(), 0);
- return dataMapRow;
- }
-
- private byte[] getColumnSchemaBinary() {
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
- return unsafeRow.getByteArray(SCHEMA);
- }
-
- /**
- * Convert schema to binary
- */
- private byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- DataOutput dataOutput = new DataOutputStream(stream);
- dataOutput.writeShort(columnSchemas.size());
- for (ColumnSchema columnSchema : columnSchemas) {
- if (columnSchema.getColumnReferenceId() == null) {
- columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
- }
- columnSchema.write(dataOutput);
- }
- byte[] byteArray = stream.toByteArray();
- // Compress with snappy to reduce the size of schema
- return Snappy.rawCompress(byteArray, byteArray.length);
- }
-
- @Override
- public void clear() {
- if (unsafeMemoryDMStore != null) {
- unsafeMemoryDMStore.freeMemory();
- unsafeMemoryDMStore = null;
- segmentProperties = null;
- }
- // clear task min/max unsafe memory
- if (null != unsafeMemorySummaryDMStore) {
- unsafeMemorySummaryDMStore.freeMemory();
- unsafeMemorySummaryDMStore = null;
- }
- }
-
- @Override
- public long getFileTimeStamp() {
- return 0;
- }
-
- @Override
- public int getAccessCount() {
- return 0;
- }
-
- @Override
- public long getMemorySize() {
- long memoryUsed = 0L;
- if (unsafeMemoryDMStore != null) {
- memoryUsed += unsafeMemoryDMStore.getMemoryUsed();
- }
- if (null != unsafeMemorySummaryDMStore) {
- memoryUsed += unsafeMemorySummaryDMStore.getMemoryUsed();
- }
- return memoryUsed;
- }
-
- public SegmentProperties getSegmentProperties() {
- return segmentProperties;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
deleted file mode 100644
index 43df813..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
-import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
-import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.SegmentFileStore;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.events.Event;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-
-/**
- * Table map for blocklet
- */
-public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
- implements BlockletDetailsFetcher, SegmentPropertiesFetcher {
-
- private static final String NAME = "clustered.btree.blocklet";
-
- public static final DataMapSchema DATA_MAP_SCHEMA =
- new DataMapSchema(NAME, BlockletDataMapFactory.class.getName());
-
- private AbsoluteTableIdentifier identifier;
-
- // segmentId -> list of index file
- private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
-
- private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainDataMap> cache;
-
- @Override
- public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
- this.identifier = identifier;
- cache = CacheProvider.getInstance()
- .createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
- }
-
- @Override
- public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public List<AbstractCoarseGrainDataMap> getDataMaps(Segment segment) throws IOException {
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- return cache.getAll(tableBlockIndexUniqueIdentifiers);
- }
-
- private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(
- Segment segment) throws IOException {
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- segmentMap.get(segment.getSegmentNo());
- if (tableBlockIndexUniqueIdentifiers == null) {
- tableBlockIndexUniqueIdentifiers = new ArrayList<>();
- Map<String, String> indexFiles;
- if (segment.getSegmentFileName() == null) {
- String path =
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
- indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
- } else {
- SegmentFileStore fileStore =
- new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
- indexFiles = fileStore.getIndexFiles();
- }
- for (Map.Entry<String, String> indexFileEntry: indexFiles.entrySet()) {
- Path indexFile = new Path(indexFileEntry.getKey());
- tableBlockIndexUniqueIdentifiers.add(
- new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(),
- indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
- }
- segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
- }
- return tableBlockIndexUniqueIdentifiers;
- }
-
- /**
- * Get the blocklet detail information based on blockletid, blockid and segmentid. This method is
- * exclusively for BlockletDataMapFactory as detail information is only available in this default
- * datamap.
- */
- @Override
- public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
- throws IOException {
- List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
- // If it is already detailed blocklet then type cast and return same
- if (blocklets.size() > 0 && blocklets.get(0) instanceof ExtendedBlocklet) {
- for (Blocklet blocklet : blocklets) {
- detailedBlocklets.add((ExtendedBlocklet) blocklet);
- }
- return detailedBlocklets;
- }
- List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- // Retrieve each blocklets detail information from blocklet datamap
- for (Blocklet blocklet : blocklets) {
- detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
- }
- return detailedBlocklets;
- }
-
- @Override
- public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment)
- throws IOException {
- if (blocklet instanceof ExtendedBlocklet) {
- return (ExtendedBlocklet) blocklet;
- }
- List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- return getExtendedBlocklet(identifiers, blocklet);
- }
-
- private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
- Blocklet blocklet) throws IOException {
- String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
- for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
- if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
- DataMap dataMap = cache.get(identifier);
- return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
- }
- }
- throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
- }
-
-
- @Override
- public List<DataMapDistributable> toDistributable(Segment segment) {
- List<DataMapDistributable> distributables = new ArrayList<>();
- try {
- CarbonFile[] carbonIndexFiles;
- if (segment.getSegmentFileName() == null) {
- carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
- } else {
- SegmentFileStore fileStore =
- new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
- Map<String, String> indexFiles = fileStore.getIndexFiles();
- carbonIndexFiles = new CarbonFile[indexFiles.size()];
- int i = 0;
- for (String indexFile : indexFiles.keySet()) {
- carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
- }
- }
- for (int i = 0; i < carbonIndexFiles.length; i++) {
- Path path = new Path(carbonIndexFiles[i].getPath());
-
- FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
- LocatedFileStatus fileStatus = iter.next();
- String[] location = fileStatus.getBlockLocations()[0].getHosts();
- BlockletDataMapDistributable distributable =
- new BlockletDataMapDistributable(path.toString());
- distributable.setLocations(location);
- distributables.add(distributable);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return distributables;
- }
-
- @Override
- public void fireEvent(Event event) {
-
- }
-
- @Override
- public void clear(Segment segment) {
- List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
- if (blockIndexes != null) {
- for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
- DataMap dataMap = cache.getIfPresent(blockIndex);
- if (dataMap != null) {
- cache.invalidate(blockIndex);
- dataMap.clear();
- }
- }
- }
- }
-
- @Override
- public void clear() {
- for (String segmentId : segmentMap.keySet().toArray(new String[segmentMap.size()])) {
- clear(new Segment(segmentId, null));
- }
- }
-
- @Override
- public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
- throws IOException {
- BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
- List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
- Path indexPath = new Path(mapDistributable.getFilePath());
- String segmentNo = mapDistributable.getSegment().getSegmentNo();
- if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
- String parent = indexPath.getParent().toString();
- identifiers
- .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
- } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
- SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
- CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
- String parentPath = carbonFile.getParentFile().getAbsolutePath();
- List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
- for (String indexFile : indexFiles) {
- identifiers.add(
- new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
- segmentNo));
- }
- }
- List<AbstractCoarseGrainDataMap> dataMaps;
- try {
- dataMaps = cache.getAll(identifiers);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return dataMaps;
- }
-
- @Override
- public DataMapMeta getMeta() {
- // TODO: pass SORT_COLUMNS into this class
- return null;
- }
-
- @Override public SegmentProperties getSegmentProperties(Segment segment) throws IOException {
- List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segment);
- assert (dataMaps.size() > 0);
- AbstractCoarseGrainDataMap coarseGrainDataMap = dataMaps.get(0);
- assert (coarseGrainDataMap instanceof BlockletDataMap);
- BlockletDataMap dataMap = (BlockletDataMap) coarseGrainDataMap;
- return dataMap.getSegmentProperties();
- }
-
- @Override public List<Blocklet> getAllBlocklets(Segment segment, List<PartitionSpec> partitions)
- throws IOException {
- List<Blocklet> blocklets = new ArrayList<>();
- List<AbstractCoarseGrainDataMap> dataMaps = getDataMaps(segment);
- for (AbstractCoarseGrainDataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(null, getSegmentProperties(segment), partitions));
- }
- return blocklets;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
index ebeb278..018c3f7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapModel.java
@@ -22,7 +22,7 @@ import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.indexstore.BlockMetaInfo;
/**
- * It is the model object to keep the information to build or initialize BlockletDataMap.
+ * It is the model object to keep the information to build or initialize BlockletIndexDataMap.
*/
public class BlockletDataMapModel extends DataMapModel {
[50/54] [abbrv] carbondata git commit: [HOTFIX] Add dava doc for
datamap interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
deleted file mode 100644
index 72a88ea..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGIndexDataMapTestCase.scala
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainIndexDataMap, AbstractCoarseGrainIndexDataMapFactory}
-import org.apache.carbondata.core.datastore.FileReader
-import org.apache.carbondata.core.datastore.block.SegmentProperties
-import org.apache.carbondata.core.datastore.compression.SnappyCompressor
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
-import org.apache.carbondata.core.util.ByteUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
-
-class CGIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory {
- var identifier: AbsoluteTableIdentifier = _
- var dataMapSchema: DataMapSchema = _
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- this.dataMapSchema = dataMapSchema
- }
-
- /**
- * Return a new write for this datamap
- */
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new CGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
- }
-
- /**
- * Get the datamap for segmentid
- */
- override def getDataMaps(segment: Segment): java.util.List[AbstractCoarseGrainDataMap] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map {f =>
- val dataMap: AbstractCoarseGrainIndexDataMap = new CGIndexDataMap()
- dataMap.init(new DataMapModel(f.getCanonicalPath))
- dataMap
- }.toList.asJava
- }
-
-
- /**
- * Get datamaps for distributable object.
- */
- override def getDataMaps(
- distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainIndexDataMap] = {
- val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
- val dataMap: AbstractCoarseGrainIndexDataMap = new CGIndexDataMap()
- dataMap.init(new DataMapModel(mapDistributable.getFilePath))
- Seq(dataMap).asJava
- }
-
- /**
- *
- * @param event
- */
- override def fireEvent(event: Event): Unit = {
- ???
- }
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
- d
- }.toList.asJava
- }
-
-
- /**
- * Clears datamap of the segment
- */
- override def clear(segment: Segment): Unit = {
-
- }
-
- /**
- * Clear all datamaps from memory
- */
- override def clear(): Unit = {
-
- }
-
- /**
- * Return metadata of this datamap
- */
- override def getMeta: DataMapMeta = {
- new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
- List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
- }
-}
-
-class CGIndexDataMap extends AbstractCoarseGrainIndexDataMap {
-
- var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
- var FileReader: FileReader = _
- var filePath: String = _
- val compressor = new SnappyCompressor
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- override def init(dataMapModel: DataMapModel): Unit = {
- this.filePath = dataMapModel.getFilePath
- val size = FileFactory.getCarbonFile(filePath).getSize
- FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
- val footerLen = FileReader.readInt(filePath, size-4)
- val bytes = FileReader.readByteArray(filePath, size-footerLen-4, footerLen)
- val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(in)
- maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
- }
-
- /**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- override def prune(
- filterExp: FilterResolverIntf,
- segmentProperties: SegmentProperties,
- partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
- val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
- val expression = filterExp.getFilterExpression
- getEqualToExpression(expression, buffer)
- val value = buffer.map { f =>
- f.getChildren.get(1).evaluate(null).getString
- }
- val meta = findMeta(value(0).getBytes)
- meta.map { f=>
- new Blocklet(f._1, f._2 + "")
- }.asJava
- }
-
-
- private def findMeta(value: Array[Byte]) = {
- val tuples = maxMin.filter { f =>
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
- }
- tuples
- }
-
- private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.isInstanceOf[EqualToExpression]) {
- buffer += expression
- } else {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
- }
- getEqualToExpression(f, buffer)
- }
- }
- }
- }
-
- /**
- * Clear complete index table and release memory.
- */
- override def clear() = {
- ???
- }
-
- override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
-}
-
-class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segment: Segment,
- dataWritePath: String,
- dataMapSchema: DataMapSchema)
- extends AbstractDataMapWriter(identifier, segment, dataWritePath) {
-
- var currentBlockId: String = null
- val cgwritepath = dataWritePath + "/" +
- dataMapSchema.getDataMapName + System.nanoTime() + ".datamap"
- lazy val stream: DataOutputStream = FileFactory
- .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
- val blockletList = new ArrayBuffer[Array[Byte]]()
- val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
- val compressor = new SnappyCompressor
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String): Unit = {
- currentBlockId = blockId
- }
-
- /**
- * End of block notification
- */
- override def onBlockEnd(blockId: String): Unit = {
-
- }
-
- /**
- * Start of new blocklet notification.
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletStart(blockletId: Int): Unit = {
-
- }
-
- /**
- * End of blocklet notification
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletEnd(blockletId: Int): Unit = {
- val sorted = blockletList
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
- maxMin +=
- ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
- blockletList.clear()
- }
-
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in IndexDataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- override def onPageAdded(blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- val size = pages(0).getPageSize
- val list = new ArrayBuffer[Array[Byte]]()
- var i = 0
- while (i < size) {
- val bytes = pages(0).getBytes(i)
- val newBytes = new Array[Byte](bytes.length - 2)
- System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
- list += newBytes
- i = i + 1
- }
- // Sort based on the column data in order to create index.
- val sorted = list
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
- blockletList += sorted.head
- blockletList += sorted.last
- }
-
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish(): Unit = {
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(maxMin)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- stream.writeInt(bytes.length)
- stream.close()
- commitFile(cgwritepath)
- }
-
-
-}
-
-class CGIndexDataMapTestCase extends QueryTest with BeforeAndAfterAll {
-
- val file2 = resourcesPath + "/compaction/fil2.csv"
- override protected def beforeAll(): Unit = {
- //n should be about 5000000 of reset if size is default 1024
- val n = 150000
- CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
- sql("DROP TABLE IF EXISTS normal_test")
- sql(
- """
- | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
- }
-
- test("test cg datamap") {
- sql("DROP TABLE IF EXISTS datamap_test_cg")
- sql(
- """
- | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
- // register datamap writer
- sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
- sql("select * from normal_test where name='n502670'"))
- }
-
- test("test cg datamap with 2 datamaps ") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(s"create datamap ggdatamap1 on table datamap_test using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
- sql(s"create datamap ggdatamap2 on table datamap_test using '${classOf[CGIndexDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='city')")
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
- }
-
- override protected def afterAll(): Unit = {
- CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
- sql("DROP TABLE IF EXISTS normal_test")
- sql("DROP TABLE IF EXISTS datamap_test_cg")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
new file mode 100644
index 0000000..d5cae15
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, SaveMode}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
+import org.apache.carbondata.core.datamap.dev.DataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{CoarseGrainDataMap, CoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.events.Event
+
+class C2DataMapFactory() extends CoarseGrainDataMapFactory {
+
+ var identifier: AbsoluteTableIdentifier = _
+
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ }
+
+ override def fireEvent(event: Event): Unit = ???
+
+ override def clear(segment: Segment): Unit = {}
+
+ override def clear(): Unit = {}
+
+ override def getDataMaps(distributable: DataMapDistributable): util.List[CoarseGrainDataMap] = ???
+
+ override def getDataMaps(segment: Segment): util.List[CoarseGrainDataMap] = ???
+
+ override def createWriter(segment: Segment, dataWritePath: String): DataMapWriter =
+ DataMapWriterSuite.dataMapWriterC2Mock(identifier, segment, dataWritePath)
+
+ override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = {
+ ???
+ }
+
+}
+
+class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
+ def buildTestData(numRows: Int): DataFrame = {
+ import sqlContext.implicits._
+ sqlContext.sparkContext.parallelize(1 to numRows, 1)
+ .map(x => ("a" + x, "b", x))
+ .toDF("c1", "c2", "c3")
+ }
+
+ def dropTable(): Unit = {
+ sql("DROP TABLE IF EXISTS carbon1")
+ sql("DROP TABLE IF EXISTS carbon2")
+ }
+
+ override def beforeAll {
+ dropTable()
+ }
+
+ test("test write datamap 2 pages") {
+ sql(s"CREATE TABLE carbon1(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
+ // register datamap writer
+ sql(s"CREATE DATAMAP test ON TABLE carbon1 USING '${classOf[C2DataMapFactory].getName}'")
+ val df = buildTestData(33000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon1")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
+ assert(
+ DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "blocklet end: 0"
+ ))
+ DataMapWriterSuite.callbackSeq = Seq()
+ }
+
+ test("test write datamap 2 blocklet") {
+ sql(s"CREATE TABLE carbon2(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
+ sql(s"CREATE DATAMAP test ON TABLE carbon2 USING '${classOf[C2DataMapFactory].getName}'")
+
+ CarbonProperties.getInstance()
+ .addProperty("carbon.blockletgroup.size.in.mb", "1")
+ CarbonProperties.getInstance()
+ .addProperty("carbon.number.of.cores.while.loading",
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
+
+ val df = buildTestData(300000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon2")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .option("SORT_SCOPE","GLOBAL_SORT")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(DataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(DataMapWriterSuite.callbackSeq.last.contains("block end"))
+ // corrected test case the min "carbon.blockletgroup.size.in.mb" size could not be less than
+ // 64 MB
+ assert(
+ DataMapWriterSuite.callbackSeq.slice(1, DataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "add page data: blocklet 0, page 2",
+ "add page data: blocklet 0, page 3",
+ "add page data: blocklet 0, page 4",
+ "add page data: blocklet 0, page 5",
+ "add page data: blocklet 0, page 6",
+ "add page data: blocklet 0, page 7",
+ "add page data: blocklet 0, page 8",
+ "add page data: blocklet 0, page 9",
+ "blocklet end: 0"
+ ))
+ DataMapWriterSuite.callbackSeq = Seq()
+ }
+
+ override def afterAll {
+ dropTable()
+ }
+}
+
+object DataMapWriterSuite {
+
+ var callbackSeq: Seq[String] = Seq[String]()
+
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segment: Segment,
+ dataWritePath: String) =
+ new DataMapWriter(identifier, segment, dataWritePath) {
+
+ override def onPageAdded(
+ blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ assert(pages.length == 1)
+ assert(pages(0).getDataType == DataTypes.STRING)
+ val bytes: Array[Byte] = pages(0).getByteArrayPage()(0)
+ assert(bytes.sameElements(Seq(0, 1, 'b'.toByte)))
+ callbackSeq :+= s"add page data: blocklet $blockletId, page $pageId"
+ }
+
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ callbackSeq :+= s"blocklet end: $blockletId"
+ }
+
+ override def onBlockEnd(blockId: String): Unit = {
+ callbackSeq :+= s"block end $blockId"
+ }
+
+ override def onBlockletStart(blockletId: Int): Unit = {
+ callbackSeq :+= s"blocklet start $blockletId"
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String) = {
+ callbackSeq :+= s"block start $blockId"
+ }
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish() = {
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
new file mode 100644
index 0000000..977b31d
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -0,0 +1,473 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.dev.{DataMapModel}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
+import org.apache.carbondata.core.datamap.dev.fgdatamap.{FineGrainBlocklet, FineGrainDataMap, FineGrainDataMapFactory}
+import org.apache.carbondata.core.datamap.dev.{DataMapModel, DataMapWriter}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class FGDataMapFactory extends FineGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapSchema: DataMapSchema = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ this.dataMapSchema = dataMapSchema
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segment: Segment, dataWritePath: String): DataMapWriter = {
+ new FGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segment: Segment): java.util.List[FineGrainDataMap] = {
+ val file = FileFactory
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val dataMap: FineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+ /**
+ * Get datamap for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[FineGrainDataMap]= {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: FineGrainDataMap = new FGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event):Unit = {
+ ???
+ }
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segment: Segment): Unit = {
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
+ }
+}
+
+class FGDataMap extends FineGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size - 4)
+ val bytes = FileReader.readByteArray(filePath, size - footerLen - 4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f =>
+ readAndFindData(f, value(0).getBytes())
+ }.filter(_.isDefined).map(_.get).asJava
+ }
+
+ private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
+ value: Array[Byte]): Option[Blocklet] = {
+ val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
+ val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(outputStream)
+ val blockletsData = obj.readObject()
+ .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
+
+ import scala.collection.Searching._
+ val searching = blockletsData
+ .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
+ (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
+ override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
+ y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
+ }
+ })
+ if (searching.insertionPoint >= 0) {
+ val f = blockletsData(searching.insertionPoint)
+ val pages = f._3.zipWithIndex.map { p =>
+ val pg = new FineGrainBlocklet.Page
+ pg.setPageId(p._1)
+ pg.setRowId(f._2(p._2).toArray)
+ pg
+ }
+ Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
+ } else {
+ None
+ }
+ }
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
+ }
+ tuples
+ }
+
+ def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear():Unit = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segment: Segment, dataWriterPath: String, dataMapSchema: DataMapSchema)
+ extends DataMapWriter(identifier, segment, dataWriterPath) {
+
+ var currentBlockId: String = null
+ val fgwritepath = dataWriterPath + "/" + dataMapSchema.getDataMapName + System.nanoTime() +
+ ".datamap"
+ val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
+ val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
+ var position: Long = 0
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
+ var addedLast: Boolean = false
+ val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
+ addedLast = false
+ } else {
+ blockletListUpdated += oldValue
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), f._3)
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletListUpdated += oldValue
+ }
+
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(blockletListUpdated)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ maxMin +=
+ ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
+ ._1), position, bytes.length))
+ position += bytes.length
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[(Array[Byte], Int)]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += ((newBytes, i))
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
+ var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
+ var addedLast: Boolean = false
+ // Merge all same column values to single row.
+ sorted.foreach { f =>
+ if (oldValue != null) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
+ oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
+ addedLast = false
+ } else {
+ blockletList += oldValue
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = true
+ }
+ } else {
+ oldValue = (f._1, Seq(f._2), Seq(pageId))
+ addedLast = false
+ }
+ }
+ if (!addedLast && oldValue != null) {
+ blockletList += oldValue
+ }
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(fgwritepath)
+ }
+}
+
+class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test fg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ test("test fg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap1 ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap2 ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='city')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
deleted file mode 100644
index d92efe7..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGIndexDataMapTestCase.scala
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainIndexDataMap, AbstractFineGrainIndexDataMapFactory}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
-import org.apache.carbondata.core.datastore.FileReader
-import org.apache.carbondata.core.datastore.block.SegmentProperties
-import org.apache.carbondata.core.datastore.compression.SnappyCompressor
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.{Blocklet, FineGrainBlocklet, PartitionSpec}
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
-import org.apache.carbondata.core.util.ByteUtil
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
-
-class FGIndexDataMapFactory extends AbstractFineGrainIndexDataMapFactory {
- var identifier: AbsoluteTableIdentifier = _
- var dataMapSchema: DataMapSchema = _
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- this.dataMapSchema = dataMapSchema
- }
-
- /**
- * Return a new write for this datamap
- */
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new FGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
- }
-
- /**
- * Get the datamap for segmentid
- */
- override def getDataMaps(segment: Segment): java.util.List[AbstractFineGrainIndexDataMap] = {
- val file = FileFactory
- .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val dataMap: AbstractFineGrainIndexDataMap = new FGIndexDataMap()
- dataMap.init(new DataMapModel(f.getCanonicalPath))
- dataMap
- }.toList.asJava
- }
-
- /**
- * Get datamap for distributable object.
- */
- override def getDataMaps(
- distributable: DataMapDistributable): java.util.List[AbstractFineGrainIndexDataMap]= {
- val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
- val dataMap: AbstractFineGrainIndexDataMap = new FGIndexDataMap()
- dataMap.init(new DataMapModel(mapDistributable.getFilePath))
- Seq(dataMap).asJava
- }
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
- val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
-
- val files = file.listFiles(new CarbonFileFilter {
- override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
- })
- files.map { f =>
- val d: DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
- d
- }.toList.asJava
- }
-
- /**
- *
- * @param event
- */
- override def fireEvent(event: Event):Unit = {
- ???
- }
-
- /**
- * Clears datamap of the segment
- */
- override def clear(segment: Segment): Unit = {
- }
-
- /**
- * Clear all datamaps from memory
- */
- override def clear(): Unit = {
- }
-
- /**
- * Return metadata of this datamap
- */
- override def getMeta: DataMapMeta = {
- new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
- List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
- }
-}
-
-class FGIndexDataMap extends AbstractFineGrainIndexDataMap {
-
- var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)] = _
- var FileReader: FileReader = _
- var filePath: String = _
- val compressor = new SnappyCompressor
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- override def init(dataMapModel: DataMapModel): Unit = {
- this.filePath = dataMapModel.getFilePath
- val size = FileFactory.getCarbonFile(filePath).getSize
- FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
- val footerLen = FileReader.readInt(filePath, size - 4)
- val bytes = FileReader.readByteArray(filePath, size - footerLen - 4, footerLen)
- val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(in)
- maxMin = obj.readObject()
- .asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]]
- }
-
- /**
- * Prune the datamap with filter expression. It returns the list of
- * blocklets where these filters can exist.
- *
- * @param filterExp
- * @return
- */
- override def prune(
- filterExp: FilterResolverIntf,
- segmentProperties: SegmentProperties,
- partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
- val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
- val expression = filterExp.getFilterExpression
- getEqualToExpression(expression, buffer)
- val value = buffer.map { f =>
- f.getChildren.get(1).evaluate(null).getString
- }
- val meta = findMeta(value(0).getBytes)
- meta.map { f =>
- readAndFindData(f, value(0).getBytes())
- }.filter(_.isDefined).map(_.get).asJava
- }
-
- private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
- value: Array[Byte]): Option[Blocklet] = {
- val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
- val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
- val obj = new ObjectInputStream(outputStream)
- val blockletsData = obj.readObject()
- .asInstanceOf[ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]]
-
- import scala.collection.Searching._
- val searching = blockletsData
- .search[(Array[Byte], Seq[Seq[Int]], Seq[Int])]((value, Seq(Seq(0)), Seq(0)))(new Ordering[
- (Array[Byte], Seq[Seq[Int]], Seq[Int])] {
- override def compare(x: (Array[Byte], Seq[Seq[Int]], Seq[Int]),
- y: (Array[Byte], Seq[Seq[Int]], Seq[Int])) = {
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(x._1, y._1)
- }
- })
- if (searching.insertionPoint >= 0) {
- val f = blockletsData(searching.insertionPoint)
- val pages = f._3.zipWithIndex.map { p =>
- val pg = new FineGrainBlocklet.Page
- pg.setPageId(p._1)
- pg.setRowId(f._2(p._2).toArray)
- pg
- }
- Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
- } else {
- None
- }
- }
-
- private def findMeta(value: Array[Byte]) = {
- val tuples = maxMin.filter { f =>
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) >= 0 &&
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) <= 0
- }
- tuples
- }
-
- def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.isInstanceOf[EqualToExpression]) {
- buffer += expression
- } else {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
- }
- getEqualToExpression(f, buffer)
- }
- }
- }
- }
-
- /**
- * Clear complete index table and release memory.
- */
- override def clear():Unit = {
- ???
- }
-
- override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
-}
-
-class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segment: Segment, dataWriterPath: String, dataMapSchema: DataMapSchema)
- extends AbstractDataMapWriter(identifier, segment, dataWriterPath) {
-
- var currentBlockId: String = null
- val fgwritepath = dataWriterPath + "/" + dataMapSchema.getDataMapName + System.nanoTime() +
- ".datamap"
- val stream: DataOutputStream = FileFactory
- .getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
- val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
- val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]), Long, Int)]()
- var position: Long = 0
- val compressor = new SnappyCompressor
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String): Unit = {
- currentBlockId = blockId
- }
-
- /**
- * End of block notification
- */
- override def onBlockEnd(blockId: String): Unit = {
-
- }
-
- /**
- * Start of new blocklet notification.
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletStart(blockletId: Int): Unit = {
-
- }
-
- /**
- * End of blocklet notification
- *
- * @param blockletId sequence number of blocklet in the block
- */
- override def onBlockletEnd(blockletId: Int): Unit = {
- val sorted = blockletList
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
- var oldValue: (Array[Byte], Seq[Seq[Int]], Seq[Int]) = null
- var addedLast: Boolean = false
- val blockletListUpdated = new ArrayBuffer[(Array[Byte], Seq[Seq[Int]], Seq[Int])]()
- // Merge all same column values to single row.
- sorted.foreach { f =>
- if (oldValue != null) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
- oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3 ++ f._3)
- addedLast = false
- } else {
- blockletListUpdated += oldValue
- oldValue = (f._1, Seq(f._2), f._3)
- addedLast = true
- }
- } else {
- oldValue = (f._1, Seq(f._2), f._3)
- addedLast = false
- }
- }
- if (!addedLast && oldValue != null) {
- blockletListUpdated += oldValue
- }
-
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(blockletListUpdated)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- maxMin +=
- ((currentBlockId + "", blockletId, (blockletListUpdated.head._1, blockletListUpdated.last
- ._1), position, bytes.length))
- position += bytes.length
- blockletList.clear()
- }
-
- /**
- * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in IndexDataMapFactory.
- *
- * Implementation should copy the content of `pages` as needed, because `pages` memory
- * may be freed after this method returns, if using unsafe column page.
- */
- override def onPageAdded(blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- val size = pages(0).getPageSize
- val list = new ArrayBuffer[(Array[Byte], Int)]()
- var i = 0
- while (i < size) {
- val bytes = pages(0).getBytes(i)
- val newBytes = new Array[Byte](bytes.length - 2)
- System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
- list += ((newBytes, i))
- i = i + 1
- }
- // Sort based on the column data in order to create index.
- val sorted = list
- .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l._1, r._1) <= 0)
- var oldValue: (Array[Byte], Seq[Int], Seq[Int]) = null
- var addedLast: Boolean = false
- // Merge all same column values to single row.
- sorted.foreach { f =>
- if (oldValue != null) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(f._1, oldValue._1) == 0) {
- oldValue = (oldValue._1, oldValue._2 ++ Seq(f._2), oldValue._3)
- addedLast = false
- } else {
- blockletList += oldValue
- oldValue = (f._1, Seq(f._2), Seq(pageId))
- addedLast = true
- }
- } else {
- oldValue = (f._1, Seq(f._2), Seq(pageId))
- addedLast = false
- }
- }
- if (!addedLast && oldValue != null) {
- blockletList += oldValue
- }
- }
-
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish(): Unit = {
- val out = new ByteOutputStream()
- val outStream = new ObjectOutputStream(out)
- outStream.writeObject(maxMin)
- outStream.close()
- val bytes = compressor.compressByte(out.getBytes)
- stream.write(bytes)
- stream.writeInt(bytes.length)
- stream.close()
- commitFile(fgwritepath)
- }
-}
-
-class FGIndexDataMapTestCase extends QueryTest with BeforeAndAfterAll {
-
- val file2 = resourcesPath + "/compaction/fil2.csv"
-
- override protected def beforeAll(): Unit = {
- //n should be about 5000000 of reset if size is default 1024
- val n = 150000
- CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
- sql("DROP TABLE IF EXISTS normal_test")
- sql(
- """
- | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
- }
-
- test("test fg datamap") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(
- s"""
- | CREATE DATAMAP ggdatamap ON TABLE datamap_test
- | USING '${classOf[FGIndexDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='name')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670'"),
- sql("select * from normal_test where name='n502670'"))
- }
-
- test("test fg datamap with 2 datamaps ") {
- sql("DROP TABLE IF EXISTS datamap_test")
- sql(
- """
- | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
- // register datamap writer
- sql(
- s"""
- | CREATE DATAMAP ggdatamap1 ON TABLE datamap_test
- | USING '${classOf[FGIndexDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='name')
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP ggdatamap2 ON TABLE datamap_test
- | USING '${classOf[FGIndexDataMapFactory].getName}'
- | DMPROPERTIES('indexcolumns'='city')
- """.stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
- checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
- }
-
- override protected def afterAll(): Unit = {
- CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
- sql("DROP TABLE IF EXISTS normal_test")
- sql("DROP TABLE IF EXISTS datamap_test")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
deleted file mode 100644
index 795ef6a..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.util
-
-import scala.collection.JavaConverters._
-
-import org.apache.spark.sql.test.util.QueryTest
-import org.apache.spark.sql.{DataFrame, SaveMode}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter}
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
-import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainIndexDataMap, AbstractCoarseGrainIndexDataMapFactory}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
-import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.events.Event
-
-class C2IndexDataMapFactory() extends AbstractCoarseGrainIndexDataMapFactory {
-
- var identifier: AbsoluteTableIdentifier = _
-
- override def init(identifier: AbsoluteTableIdentifier,
- dataMapSchema: DataMapSchema): Unit = {
- this.identifier = identifier
- }
-
- override def fireEvent(event: Event): Unit = ???
-
- override def clear(segment: Segment): Unit = {}
-
- override def clear(): Unit = {}
-
- override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainIndexDataMap] = ???
-
- override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainIndexDataMap] = ???
-
- override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter =
- IndexDataMapWriterSuite.dataMapWriterC2Mock(identifier, segment, dataWritePath)
-
- override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
-
- /**
- * Get all distributable objects of a segmentid
- *
- * @return
- */
- override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = {
- ???
- }
-
-}
-
-class IndexDataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
- def buildTestData(numRows: Int): DataFrame = {
- import sqlContext.implicits._
- sqlContext.sparkContext.parallelize(1 to numRows, 1)
- .map(x => ("a" + x, "b", x))
- .toDF("c1", "c2", "c3")
- }
-
- def dropTable(): Unit = {
- sql("DROP TABLE IF EXISTS carbon1")
- sql("DROP TABLE IF EXISTS carbon2")
- }
-
- override def beforeAll {
- dropTable()
- }
-
- test("test write datamap 2 pages") {
- sql(s"CREATE TABLE carbon1(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
- // register datamap writer
- sql(s"CREATE DATAMAP test ON TABLE carbon1 USING '${classOf[C2IndexDataMapFactory].getName}'")
- val df = buildTestData(33000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon1")
- .option("tempCSV", "false")
- .option("sort_columns","c1")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(IndexDataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(IndexDataMapWriterSuite.callbackSeq.last.contains("block end"))
- assert(
- IndexDataMapWriterSuite.callbackSeq.slice(1, IndexDataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "blocklet end: 0"
- ))
- IndexDataMapWriterSuite.callbackSeq = Seq()
- }
-
- test("test write datamap 2 blocklet") {
- sql(s"CREATE TABLE carbon2(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
- sql(s"CREATE DATAMAP test ON TABLE carbon2 USING '${classOf[C2IndexDataMapFactory].getName}'")
-
- CarbonProperties.getInstance()
- .addProperty("carbon.blockletgroup.size.in.mb", "1")
- CarbonProperties.getInstance()
- .addProperty("carbon.number.of.cores.while.loading",
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
-
- val df = buildTestData(300000)
-
- // save dataframe to carbon file
- df.write
- .format("carbondata")
- .option("tableName", "carbon2")
- .option("tempCSV", "false")
- .option("sort_columns","c1")
- .option("SORT_SCOPE","GLOBAL_SORT")
- .mode(SaveMode.Overwrite)
- .save()
-
- assert(IndexDataMapWriterSuite.callbackSeq.head.contains("block start"))
- assert(IndexDataMapWriterSuite.callbackSeq.last.contains("block end"))
- // corrected test case the min "carbon.blockletgroup.size.in.mb" size could not be less than
- // 64 MB
- assert(
- IndexDataMapWriterSuite.callbackSeq.slice(1, IndexDataMapWriterSuite.callbackSeq.length - 1) == Seq(
- "blocklet start 0",
- "add page data: blocklet 0, page 0",
- "add page data: blocklet 0, page 1",
- "add page data: blocklet 0, page 2",
- "add page data: blocklet 0, page 3",
- "add page data: blocklet 0, page 4",
- "add page data: blocklet 0, page 5",
- "add page data: blocklet 0, page 6",
- "add page data: blocklet 0, page 7",
- "add page data: blocklet 0, page 8",
- "add page data: blocklet 0, page 9",
- "blocklet end: 0"
- ))
- IndexDataMapWriterSuite.callbackSeq = Seq()
- }
-
- override def afterAll {
- dropTable()
- }
-}
-
-object IndexDataMapWriterSuite {
-
- var callbackSeq: Seq[String] = Seq[String]()
-
- def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segment: Segment,
- dataWritePath: String) =
- new AbstractDataMapWriter(identifier, segment, dataWritePath) {
-
- override def onPageAdded(
- blockletId: Int,
- pageId: Int,
- pages: Array[ColumnPage]): Unit = {
- assert(pages.length == 1)
- assert(pages(0).getDataType == DataTypes.STRING)
- val bytes: Array[Byte] = pages(0).getByteArrayPage()(0)
- assert(bytes.sameElements(Seq(0, 1, 'b'.toByte)))
- callbackSeq :+= s"add page data: blocklet $blockletId, page $pageId"
- }
-
- override def onBlockletEnd(blockletId: Int): Unit = {
- callbackSeq :+= s"blocklet end: $blockletId"
- }
-
- override def onBlockEnd(blockId: String): Unit = {
- callbackSeq :+= s"block end $blockId"
- }
-
- override def onBlockletStart(blockletId: Int): Unit = {
- callbackSeq :+= s"blocklet start $blockletId"
- }
-
- /**
- * Start of new block notification.
- *
- * @param blockId file name of the carbondata file
- */
- override def onBlockStart(blockId: String) = {
- callbackSeq :+= s"block start $blockId"
- }
-
- /**
- * This is called during closing of writer.So after this call no more data will be sent to this
- * class.
- */
- override def finish() = {
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
new file mode 100644
index 0000000..eaa2ae7
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{File, FilenameFilter}
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.common.exceptions.MetadataProcessException
+import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
+
+ val testData = s"$resourcesPath/sample.csv"
+
+ override def beforeAll {
+ sql("drop table if exists datamaptest")
+ sql("drop table if exists datamapshowtest")
+ sql("drop table if exists uniqdata")
+ sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
+ }
+
+ val newClass = "org.apache.spark.sql.CarbonSource"
+
+ test("test datamap create: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
+ }
+ }
+
+ test("test datamap create with dmproperties: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ }
+ }
+
+ test("test datamap create with existing name: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(
+ s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ }
+ }
+
+ test("test datamap create with preagg") {
+ sql("drop datamap if exists datamap3 on table datamaptest")
+ sql(
+ "create datamap datamap3 on table datamaptest using 'preaggregate' as select count(a) from datamaptest")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
+ assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
+ }
+
+ test("check hivemetastore after drop datamap") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ "true")
+ sql("drop table if exists hiveMetaStoreTable")
+ sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
+
+ sql(
+ "create datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable using 'preaggregate' as select count(a) from hiveMetaStoreTable")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable"), true, "datamap_hiveMetaStoreTable")
+
+ sql("drop datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable"), false, "datamap_hiveMetaStoreTable")
+
+ } finally {
+ sql("drop table hiveMetaStoreTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("drop the table having pre-aggregate") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ "true")
+ sql("drop table if exists hiveMetaStoreTable_1")
+ sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
+
+ sql(
+ "create datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1 using 'preaggregate' as select count(a) from hiveMetaStoreTable_1")
+
+ checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
+ true,
+ "datamap_hiveMetaStoreTable_1")
+
+ sql("drop datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
+ false,
+ "datamap_hiveMetaStoreTable_1")
+ assert(sql("show datamap on table hiveMetaStoreTable_1").collect().length == 0)
+ sql("drop table hiveMetaStoreTable_1")
+
+ checkExistence(sql("show tables"), false, "datamap_hiveMetaStoreTable_1")
+ }
+ finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("test datamap create with preagg with duplicate name") {
+ sql(
+ s"""
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ }
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ }
+
+ test("test drop non-exist datamap") {
+ intercept[NoSuchDataMapException] {
+ sql("drop datamap nonexist on table datamaptest")
+ }
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ }
+
+ test("test show datamap without preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' ")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
+ }
+ }
+
+ test("test show datamap with preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 2)
+ checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
+ }
+ }
+
+ test("test show datamap with no datamap") {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ assert(sql("show datamap on table datamapshowtest").collect().length == 0)
+ }
+
+ test("test show datamap after dropping datamap: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ sql("drop datamap datamap1 on table datamapshowtest")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 1)
+ checkExistence(frame, true, "datamap2", "(NA)", newClass)
+ }
+ }
+
+ test("test if preaggregate load is successfull for hivemetastore") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("DROP TABLE IF EXISTS maintable")
+ sql(
+ """
+ | CREATE TABLE maintable(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
+
+ .stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
+ checkAnswer(sql(s"select * from maintable_preagg_sum"),
+ Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ } finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("test preaggregate load for decimal column for hivemetastore") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
+ sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ }
+
+ test("create pre-agg table with path") {
+ sql("drop table if exists main_preagg")
+ sql("drop table if exists main ")
+ val warehouse = s"$metastoredb/warehouse"
+ val path = warehouse + "/" + System.nanoTime + "_preAggTestPath"
+ sql(
+ s"""
+ | create table main(
+ | year int,
+ | month int,
+ | name string,
+ | salary int)
+ | stored by 'carbondata'
+ | tblproperties('sort_columns'='month,year,name')
+ """.stripMargin)
+ sql("insert into main select 10,11,'amy',12")
+ sql("insert into main select 10,11,'amy',14")
+ sql(
+ s"""
+ | create datamap preagg
+ | on table main
+ | using 'preaggregate'
+ | dmproperties ('path'='$path')
+ | as select name,avg(salary)
+ | from main
+ | group by name
+ """.stripMargin)
+ assertResult(true)(new File(path).exists())
+ assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
+ checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
+ checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
+ sql("drop datamap preagg on table main")
+ assertResult(false)(new File(path).exists())
+ sql("drop table main")
+ }
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS maintable")
+ sql("drop table if exists uniqdata")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ sql("drop table if exists datamaptest")
+ sql("drop table if exists datamapshowtest")
+ }
+}
[11/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 8c8d08f..a689d8e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -124,22 +124,22 @@ public class UnsafeFixedLengthDimensionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// based on index we need to calculate the actual position in memory block
- index = index * columnValueSize;
+ rowId = rowId * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
compareResult = (CarbonUnsafe.getUnsafe()
- .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
+ .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + rowId)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
break;
}
- index++;
+ rowId++;
}
return compareResult;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 36b2bd8..e1eb378 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -189,11 +189,11 @@ public class UnsafeVariableLengthDimesionDataChunkStore
/**
* to compare the two byte array
*
- * @param index index of first byte array
+ * @param rowId index of first byte array
* @param compareValue value of to be compared
* @return compare result
*/
- @Override public int compareTo(int index, byte[] compareValue) {
+ @Override public int compareTo(int rowId, byte[] compareValue) {
// now to get the row from memory block we need to do following thing
// 1. first get the current offset
// 2. if it's not a last row- get the next row offset
@@ -201,13 +201,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// else subtract the current row offset
// with complete data length get the offset of set of data
int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long)index
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long) rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
- if (index < numberOfRows - 1) {
+ if (rowId < numberOfRows - 1) {
int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
index 74d268a..e2a4161 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -29,13 +29,6 @@ public class ColumnGroupModel {
private int noOfColumnsStore;
/**
- * whether given index is columnar or not
- * true: columnar
- * false: row block
- */
- private boolean[] columnarStore;
-
- /**
* column groups
* e.g
* {{0,1,2},3,4,{5,6}}
@@ -77,15 +70,6 @@ public class ColumnGroupModel {
}
/**
- * it's an identifier for row block or single column block
- *
- * @param columnarStore
- */
- public void setColumnarStore(boolean[] columnarStore) {
- this.columnarStore = columnarStore;
- }
-
- /**
* set column groups
*
* @param columnGroups
@@ -95,16 +79,6 @@ public class ColumnGroupModel {
}
/**
- * check if given column group is columnar
- *
- * @param colGroup
- * @return true if given block is columnar
- */
- public boolean isColumnar(int colGroup) {
- return columnarStore[colGroup];
- }
-
- /**
* @return columngroups
*/
public int[][] getColumnGroup() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
deleted file mode 100644
index 182c8eb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.impl;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public class DFSFileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FSDataInputStream> fileNameAndStreamCache;
-
- private String queryId;
-
- private boolean readPageByPage;
-
-
- public DFSFileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length, offset);
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- public FSDataInputStream updateCache(String filePath) throws IOException {
- FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- Path pt = new Path(filePath);
- FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
- fileChannel = fs.open(pt);
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.seek(offset);
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- @Override public int readInt(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readInt();
- }
-
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public void finish() throws IOException {
- for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
- FSDataInputStream channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length);
- }
-
- @Override public long readLong(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public int readInt(String filePath) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return fileChannel.readInt();
- }
-
- @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException {
- byte[] readByteArray = readByteArray(filePath, offset, length);
- ByteBuffer byteBuffer = ByteBuffer.wrap(readByteArray);
- byteBuffer.rewind();
- return byteBuffer;
- }
-
- @Override public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- @Override public String getQueryId() {
- return queryId;
- }
-
- @Override public void setReadPageByPage(boolean isReadPageByPage) {
- this.readPageByPage = isReadPageByPage;
- }
-
- @Override public boolean isReadPageByPage() {
- return readPageByPage;
- }
-
- public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
- return fileNameAndStreamCache;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
new file mode 100644
index 0000000..1a0cd41
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileReaderImpl.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileReader;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public class DFSFileReaderImpl implements FileReader {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FSDataInputStream> fileNameAndStreamCache;
+
+ private boolean readPageByPage;
+
+ public DFSFileReaderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length, offset);
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FSDataInputStream updateCache(String filePath) throws IOException {
+ FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ Path pt = new Path(filePath);
+ FileSystem fs = pt.getFileSystem(FileFactory.getConfiguration());
+ fileChannel = fs.open(pt);
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.seek(offset);
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readInt();
+ }
+
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public void finish() throws IOException {
+ for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
+ FSDataInputStream channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length);
+ }
+
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public int readInt(String filePath) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return fileChannel.readInt();
+ }
+
+ @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException {
+ byte[] readByteArray = readByteArray(filePath, offset, length);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(readByteArray);
+ byteBuffer.rewind();
+ return byteBuffer;
+ }
+
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+
+ public Map<String, FSDataInputStream> getFileNameAndStreamCache() {
+ return fileNameAndStreamCache;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
index 67648fe..b58a473 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DefaultFileTypeProvider.java
@@ -17,24 +17,28 @@
package org.apache.carbondata.core.datastore.impl;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.filesystem.*;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.filesystem.AlluxioCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.ViewFSCarbonFile;
import org.apache.hadoop.conf.Configuration;
public class DefaultFileTypeProvider implements FileTypeInerface {
- public FileHolder getFileHolder(FileFactory.FileType fileType) {
+ public FileReader getFileHolder(FileFactory.FileType fileType) {
switch (fileType) {
case LOCAL:
- return new FileHolderImpl();
+ return new FileReaderImpl();
case HDFS:
case ALLUXIO:
case VIEWFS:
case S3:
- return new DFSFileHolderImpl();
+ return new DFSFileReaderImpl();
default:
- return new FileHolderImpl();
+ return new FileReaderImpl();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index f141991..9bcdfae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -28,7 +28,7 @@ import java.nio.channels.FileChannel;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.commons.io.FileUtils;
@@ -62,7 +62,7 @@ public final class FileFactory {
return configuration;
}
- public static FileHolder getFileHolder(FileType fileType) {
+ public static FileReader getFileHolder(FileType fileType) {
return fileFileTypeInerface.getFileHolder(fileType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
deleted file mode 100644
index cc589b7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.impl;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.FileHolder;
-
-public class FileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FileChannel> fileNameAndStreamCache;
- private String queryId;
-
- private boolean readPageByPage;
-
- /**
- * FileHolderImpl Constructor
- * It will create the cache
- */
- public FileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- public FileHolderImpl(int capacity) {
- this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
- }
-
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length, offset);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- @Override public void finish() throws IOException {
- for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public int readInt(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- @Override public int readInt(String filePath) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FileChannel updateCache(String filePath) throws FileNotFoundException {
- FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- FileInputStream stream = new FileInputStream(filePath);
- fileChannel = stream.getChannel();
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.position(offset);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and position
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- @Override public long readLong(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException {
- ByteBuffer byteBuffer = ByteBuffer.allocate(length);
- FileChannel fileChannel = updateCache(filePath);
- fileChannel.position(offset);
- fileChannel.read(byteBuffer);
- byteBuffer.rewind();
- return byteBuffer;
- }
-
- @Override public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- @Override public String getQueryId() {
- return queryId;
- }
-
- @Override public void setReadPageByPage(boolean isReadPageByPage) {
- this.readPageByPage = isReadPageByPage;
- }
-
- @Override public boolean isReadPageByPage() {
- return readPageByPage;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
new file mode 100644
index 0000000..6fef278
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileReaderImpl.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileReader;
+
+public class FileReaderImpl implements FileReader {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FileChannel> fileNameAndStreamCache;
+
+ private boolean readPageByPage;
+
+ /**
+ * FileReaderImpl Constructor
+ * It will create the cache
+ */
+ public FileReaderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ public FileReaderImpl(int capacity) {
+ this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
+ }
+
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length, offset);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ @Override public void finish() throws IOException {
+ for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
+ FileChannel channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ @Override public int readInt(String filePath) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FileChannel updateCache(String filePath) throws FileNotFoundException {
+ FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ FileInputStream stream = new FileInputStream(filePath);
+ fileChannel = stream.getChannel();
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.position(offset);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and position
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ @Override public ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException {
+ ByteBuffer byteBuffer = ByteBuffer.allocate(length);
+ FileChannel fileChannel = updateCache(filePath);
+ fileChannel.position(offset);
+ fileChannel.read(byteBuffer);
+ byteBuffer.rewind();
+ return byteBuffer;
+ }
+
+ @Override public void setReadPageByPage(boolean isReadPageByPage) {
+ this.readPageByPage = isReadPageByPage;
+ }
+
+ @Override public boolean isReadPageByPage() {
+ return readPageByPage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
index 4676278..413261c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileTypeInerface.java
@@ -17,14 +17,14 @@
package org.apache.carbondata.core.datastore.impl;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.hadoop.conf.Configuration;
public interface FileTypeInerface {
- FileHolder getFileHolder(FileFactory.FileType fileType);
+ FileReader getFileHolder(FileFactory.FileType fileType);
CarbonFile getCarbonFile(String path, FileFactory.FileType fileType);
CarbonFile getCarbonFile(String path, FileFactory.FileType fileType, Configuration configuration);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 19b1f1c..fe4cf83 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -18,9 +18,8 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -31,38 +30,31 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public abstract class AbstractBTreeLeafNode implements BTreeNode {
/**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- protected BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- /**
* number of keys in a btree
*/
- protected int numberOfKeys;
+ int numberOfKeys;
/**
* node number
*/
- protected long nodeNumber;
+ long nodeNumber;
/**
* Next node of the leaf
*/
- protected BTreeNode nextNode;
+ private BTreeNode nextNode;
/**
* max key of the column this will be used to check whether this leaf will
* be used for scanning or not
*/
- protected byte[][] maxKeyOfColumns;
+ byte[][] maxKeyOfColumns;
/**
* min key of the column this will be used to check whether this leaf will
* be used for scanning or not
*/
- protected byte[][] minKeyOfColumns;
+ byte[][] minKeyOfColumns;
/**
* Method to get the next block this can be used while scanning when
@@ -70,7 +62,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
*
* @return next block
*/
- @Override public int nodeSize() {
+ @Override public int numRows() {
return this.numberOfKeys;
}
@@ -109,7 +101,7 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
*
* @return block number
*/
- @Override public long nodeNumber() {
+ @Override public long nodeIndex() {
return nodeNumber;
}
@@ -174,11 +166,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// dimension chunks
return null;
@@ -188,11 +180,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
+ * @param columnIndex block index to be read
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader,
- int blockIndex) throws IOException {
+ @Override public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader,
+ int columnIndex) throws IOException {
// No required here as leaf which will will be use this class will implement
// its own get dimension chunks
return null;
@@ -202,11 +194,11 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
@@ -216,30 +208,16 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
// No required here as leaf which will will be use this class will implement its own get
// measure chunks
return null;
}
- /**
- * @param deleteDeltaDataCache
- */
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
-
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
- /**
- * @return the segmentProperties
- */
- public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
@Override
public int getPageRowCount(int pageNumber) {
throw new UnsupportedOperationException("Unsupported operation");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2f8aadf..688d56a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -106,7 +106,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private BTreeNode findFirstLeafNode(IndexKey key, BTreeNode node) {
int childNodeIndex;
int low = 0;
- int high = node.nodeSize() - 1;
+ int high = node.numRows() - 1;
int mid = 0;
int compareRes = -1;
IndexKey[] nodeKeys = node.getNodeKeys();
@@ -156,7 +156,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private BTreeNode findLastLeafNode(IndexKey key, BTreeNode node) {
int childNodeIndex;
int low = 0;
- int high = node.nodeSize() - 1;
+ int high = node.numRows() - 1;
int mid = 0;
int compareRes = -1;
IndexKey[] nodeKeys = node.getNodeKeys();
@@ -172,7 +172,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
} else {
int currentPos = mid;
// if key is matched then get the first entry
- while (currentPos + 1 < node.nodeSize()
+ while (currentPos + 1 < node.numRows()
&& compareIndexes(key, nodeKeys[currentPos + 1]) == 0) {
currentPos++;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index ccc5e12..c200f8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -19,10 +19,9 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -34,13 +33,6 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public class BTreeNonLeafNode implements BTreeNode {
/**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- protected BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- /**
* Child nodes
*/
private BTreeNode[] children;
@@ -50,7 +42,7 @@ public class BTreeNonLeafNode implements BTreeNode {
*/
private List<IndexKey> listOfKeys;
- public BTreeNonLeafNode() {
+ BTreeNonLeafNode() {
// creating a list which will store all the indexes
listOfKeys = new ArrayList<IndexKey>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
@@ -120,7 +112,7 @@ public class BTreeNonLeafNode implements BTreeNode {
*
* @return number of keys in the block
*/
- @Override public int nodeSize() {
+ @Override public int numRows() {
return listOfKeys.size();
}
@@ -131,11 +123,11 @@ public class BTreeNonLeafNode implements BTreeNode {
*
* @return block number
*/
- @Override public long nodeNumber() {
+ @Override public long nodeIndex() {
throw new UnsupportedOperationException("Unsupported operation");
}
- @Override public String blockletId() {
+ @Override public short blockletIndex() {
throw new UnsupportedOperationException("Unsupported operation");
}
@@ -171,11 +163,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) {
// operation of getting the dimension chunks is not supported as its a
// non leaf node
@@ -191,8 +183,8 @@ public class BTreeNonLeafNode implements BTreeNode {
* @param fileReader file reader to read the chunk from file
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader,
- int blockIndexes) {
+ @Override public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader,
+ int columnIndex) {
// operation of getting the dimension chunk is not supported as its a
// non leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -205,11 +197,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) {
// operation of getting the measure chunk is not supported as its a non
// leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -222,11 +214,11 @@ public class BTreeNonLeafNode implements BTreeNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex) {
// operation of getting the measure chunk is not supported as its a non
// leaf node
// and in case of B+Tree data will be stored only in leaf node and
@@ -236,20 +228,6 @@ public class BTreeNonLeafNode implements BTreeNode {
}
/**
- * @return the segmentProperties
- */
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
-
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
- /**
- * @return the segmentProperties
- */
- public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
- /**
* number of pages in blocklet
* @return
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
index 25817f5..8af7eae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
@@ -38,7 +38,7 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
* node
* @param metadataIndex metadata index
*/
- public BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) {
+ BlockBTreeLeafNode(BTreeBuilderInfo builderInfos, int metadataIndex, long nodeNumber) {
DataFileFooter footer = builderInfos.getFooterList().get(metadataIndex);
BlockletMinMaxIndex minMaxIndex = footer.getBlockletIndex().getMinMaxIndex();
maxKeyOfColumns = minMaxIndex.getMaxValues();
@@ -63,8 +63,8 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method is suppose to return the Blocklet ID.
* @return
*/
- @Override public String blockletId() {
- return blockInfo.getTableBlockInfo().getDetailInfo().getBlockletId().toString();
+ @Override public short blockletIndex() {
+ return blockInfo.getTableBlockInfo().getDetailInfo().getBlockletId();
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 94221ba..ddd7fcf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
@@ -69,7 +69,7 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* this will be used during query execution when we can
* give some leaf node of a btree to one executor some to other
*/
- public BlockletBTreeLeafNode(BTreeBuilderInfo builderInfos, int leafIndex, long nodeNumber) {
+ BlockletBTreeLeafNode(BTreeBuilderInfo builderInfos, int leafIndex, long nodeNumber) {
// get a lead node min max
BlockletMinMaxIndex minMaxIndex =
builderInfos.getFooterList().get(0).getBlockletList().get(leafIndex).getBlockletIndex()
@@ -124,23 +124,23 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
}
}
- @Override public String blockletId() {
- return "0";
+ @Override public short blockletIndex() {
+ return 0;
}
/**
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
+ * @param columnIndexRange indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
if (fileReader.isReadPageByPage()) {
- return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, blockIndexes);
+ return dimensionChunksPageLevelReader.readRawDimensionChunks(fileReader, columnIndexRange);
} else {
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, columnIndexRange);
}
}
@@ -148,15 +148,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to get the dimension chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
+ * @param columnIndex block index to be read
* @return dimension data chunk
*/
- @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndex)
- throws IOException {
+ @Override public DimensionRawColumnChunk readDimensionChunk(
+ FileReader fileReader, int columnIndex) throws IOException {
if (fileReader.isReadPageByPage()) {
- return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, blockIndex);
+ return dimensionChunksPageLevelReader.readRawDimensionChunk(fileReader, columnIndex);
} else {
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, columnIndex);
}
}
@@ -164,15 +164,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
+ * @param columnIndexRange block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
- int[][] blockIndexes) throws IOException {
+ @Override public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader,
+ int[][] columnIndexRange) throws IOException {
if (fileReader.isReadPageByPage()) {
- return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, blockIndexes);
+ return measureColumnChunkPageLevelReader.readRawMeasureChunks(fileReader, columnIndexRange);
} else {
- return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, columnIndexRange);
}
}
@@ -180,15 +180,15 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
throws IOException {
if (fileReader.isReadPageByPage()) {
- return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, blockIndex);
+ return measureColumnChunkPageLevelReader.readRawMeasureChunk(fileReader, columnIndex);
} else {
- return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, columnIndex);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index 6d96b3b..597def0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -41,7 +41,13 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.Encoding;
-import static org.apache.carbondata.format.Encoding.*;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_FLOATING;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_FLOATING;
+import static org.apache.carbondata.format.Encoding.ADAPTIVE_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.BOOL_BYTE;
+import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS;
+import static org.apache.carbondata.format.Encoding.RLE_INTEGRAL;
/**
* Base class for encoding factory implementation.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
index c7411d6..daba470 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.dictionary.service.AbstractDictionaryServer;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.util.CarbonProperties;
-
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
index ce05fe2..2865d4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
@@ -213,10 +213,6 @@ public class BlockletDetailInfo implements Serializable, Writable {
return columnSchemas;
}
- public void setColumnSchemas(List<ColumnSchema> columnSchemas) {
- this.columnSchemas = columnSchemas;
- }
-
public void setColumnSchemaBinary(byte[] columnSchemaBinary) {
this.columnSchemaBinary = columnSchemaBinary;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
new file mode 100644
index 0000000..5cd59cb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.constants.CarbonVersionConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileReader;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
+import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+
+/**
+ * wrapper for blocklet data map data
+ */
+public class BlockletDataRefNode implements DataRefNode {
+
+ private List<TableBlockInfo> blockInfos;
+
+ private int index;
+
+ private int[] dimensionLens;
+
+ BlockletDataRefNode(List<TableBlockInfo> blockInfos, int index, int[] dimensionLens) {
+ this.blockInfos = blockInfos;
+ // Update row count and page count to blocklet info
+ for (TableBlockInfo blockInfo : blockInfos) {
+ BlockletDetailInfo detailInfo = blockInfo.getDetailInfo();
+ detailInfo.getBlockletInfo().setNumberOfRows(detailInfo.getRowCount());
+ detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
+ detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
+ int[] pageRowCount = new int[detailInfo.getPagesCount()];
+ int numberOfPagesCompletelyFilled = detailInfo.getRowCount();
+ // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the number
+ // of pages filled
+ if (blockInfo.getVersion() == ColumnarFormatVersion.V2) {
+ numberOfPagesCompletelyFilled /=
+ CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
+ } else {
+ numberOfPagesCompletelyFilled /=
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
+ int lastPageRowCount = detailInfo.getRowCount()
+ % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
+ pageRowCount[i] =
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+ }
+ if (lastPageRowCount > 0) {
+ pageRowCount[pageRowCount.length - 1] = lastPageRowCount;
+ }
+ detailInfo.getBlockletInfo().setNumberOfRowsPerPage(pageRowCount);
+ }
+ this.index = index;
+ this.dimensionLens = dimensionLens;
+ }
+
+ @Override public DataRefNode getNextDataRefNode() {
+ if (index + 1 < blockInfos.size()) {
+ return new BlockletDataRefNode(blockInfos, index + 1, dimensionLens);
+ }
+ return null;
+ }
+
+ @Override public int numRows() {
+ return blockInfos.get(index).getDetailInfo().getRowCount();
+ }
+
+ @Override public long nodeIndex() {
+ return index;
+ }
+
+ @Override public short blockletIndex() {
+ return blockInfos.get(index).getDetailInfo().getBlockletId();
+ }
+
+ @Override
+ public byte[][] getColumnsMaxValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMaxValues();
+ }
+ return null;
+ }
+
+ @Override
+ public byte[][] getColumnsMinValue() {
+ BlockletIndex blockletIndex =
+ blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
+ // In case of blocklet distribution this will be null
+ if (null != blockletIndex) {
+ return blockletIndex.getMinMaxIndex().getMinValues();
+ }
+ return null;
+ }
+
+ @Override
+ public DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader, int[][] blockIndexes)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ }
+
+ @Override
+ public DimensionRawColumnChunk readDimensionChunk(FileReader fileReader, int columnIndex)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, columnIndex);
+ }
+
+ @Override
+ public MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ measureColumnChunkReader.readRawMeasureChunks(fileReader, columnIndexRange);
+ updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunks);
+ return measureRawColumnChunks;
+ }
+
+ @Override public MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
+ MeasureRawColumnChunk measureRawColumnChunk =
+ measureColumnChunkReader.readRawMeasureChunk(fileReader, columnIndex);
+ updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunk);
+ return measureRawColumnChunk;
+ }
+
+ /**
+ * This method is written specifically for old store wherein the measure min and max values
+ * are written opposite (i.e min in place of max and amx in place of min). Due to this computing
+ * f measure filter with current code is impacted. In order to sync with current min and
+ * max values only in case old store and measures is reversed
+ *
+ * @param measureRawColumnChunk
+ */
+ private void updateMeasureRawColumnChunkMinMaxValues(
+ MeasureRawColumnChunk measureRawColumnChunk) {
+ if (blockInfos.get(index).isDataBlockFromOldStore()) {
+ byte[][] maxValues = measureRawColumnChunk.getMaxValues();
+ byte[][] minValues = measureRawColumnChunk.getMinValues();
+ measureRawColumnChunk.setMaxValues(minValues);
+ measureRawColumnChunk.setMinValues(maxValues);
+ }
+ }
+
+ private void updateMeasureRawColumnChunkMinMaxValues(
+ MeasureRawColumnChunk[] measureRawColumnChunks) {
+ if (blockInfos.get(index).isDataBlockFromOldStore()) {
+ for (int i = 0; i < measureRawColumnChunks.length; i++) {
+ if (null != measureRawColumnChunks[i]) {
+ updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunks[i]);
+ }
+ }
+ }
+ }
+
+ private DimensionColumnChunkReader getDimensionColumnChunkReader(FileReader fileReader) {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath(), false);
+ }
+ }
+
+ private MeasureColumnChunkReader getMeasureColumnChunkReader(FileReader fileReader) {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ if (fileReader.isReadPageByPage()) {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), true);
+ } else {
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath(), false);
+ }
+ }
+
+ @Override public int numberOfPages() {
+ return blockInfos.get(index).getDetailInfo().getPagesCount();
+ }
+
+ @Override public int getPageRowCount(int pageNumber) {
+ return blockInfos.get(index).getDetailInfo().getBlockletInfo()
+ .getNumberOfRowsPerPage()[pageNumber];
+ }
+
+ public int numberOfNodes() {
+ return blockInfos.size();
+ }
+
+ public List<TableBlockInfo> getBlockInfos() {
+ return blockInfos;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
deleted file mode 100644
index 4e49ede..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
-import org.apache.carbondata.core.constants.CarbonVersionConstants;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
-import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
-import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
-import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-
-/**
- * wrapper for blocklet data map data
- */
-public class BlockletDataRefNodeWrapper implements DataRefNode {
-
- private List<TableBlockInfo> blockInfos;
-
- private int index;
-
- private int[] dimensionLens;
-
- private BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
-
- public BlockletDataRefNodeWrapper(List<TableBlockInfo> blockInfos, int index,
- int[] dimensionLens) {
- this.blockInfos = blockInfos;
- // Update row count and page count to blocklet info
- for (TableBlockInfo blockInfo : blockInfos) {
- BlockletDetailInfo detailInfo = blockInfo.getDetailInfo();
- detailInfo.getBlockletInfo().setNumberOfRows(detailInfo.getRowCount());
- detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
- detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
- int[] pageRowCount = new int[detailInfo.getPagesCount()];
- int numberOfPagesCompletelyFilled = detailInfo.getRowCount();
- // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the number
- // of pages filled
- if (blockInfo.getVersion() == ColumnarFormatVersion.V2) {
- numberOfPagesCompletelyFilled /=
- CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
- } else {
- numberOfPagesCompletelyFilled /=
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
- int lastPageRowCount = detailInfo.getRowCount()
- % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {
- pageRowCount[i] =
- CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
- }
- if (lastPageRowCount > 0) {
- pageRowCount[pageRowCount.length - 1] = lastPageRowCount;
- }
- detailInfo.getBlockletInfo().setNumberOfRowsPerPage(pageRowCount);
- }
- this.index = index;
- this.dimensionLens = dimensionLens;
- }
-
- @Override public DataRefNode getNextDataRefNode() {
- if (index + 1 < blockInfos.size()) {
- return new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
- }
- return null;
- }
-
- @Override public int nodeSize() {
- return blockInfos.get(index).getDetailInfo().getRowCount();
- }
-
- @Override public long nodeNumber() {
- return index;
- }
-
- @Override public String blockletId() {
- return blockInfos.get(index).getDetailInfo().getBlockletId().toString();
- }
-
- @Override
- public byte[][] getColumnsMaxValue() {
- BlockletIndex blockletIndex =
- blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
- // In case of blocklet distribution this will be null
- if (null != blockletIndex) {
- return blockletIndex.getMinMaxIndex().getMaxValues();
- }
- return null;
- }
-
- @Override
- public byte[][] getColumnsMinValue() {
- BlockletIndex blockletIndex =
- blockInfos.get(index).getDetailInfo().getBlockletInfo().getBlockletIndex();
- // In case of blocklet distribution this will be null
- if (null != blockletIndex) {
- return blockletIndex.getMinMaxIndex().getMinValues();
- }
- return null;
- }
-
- @Override
- public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
- return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
- }
-
- @Override
- public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
- throws IOException {
- DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader(fileReader);
- return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndexes);
- }
-
- @Override
- public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
- throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
- MeasureRawColumnChunk[] measureRawColumnChunks =
- measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
- updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunks);
- return measureRawColumnChunks;
- }
-
- @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
- throws IOException {
- MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader(fileReader);
- MeasureRawColumnChunk measureRawColumnChunk =
- measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
- updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunk);
- return measureRawColumnChunk;
- }
-
- /**
- * This method is written specifically for old store wherein the measure min and max values
- * are written opposite (i.e min in place of max and amx in place of min). Due to this computing
- * f measure filter with current code is impacted. In order to sync with current min and
- * max values only in case old store and measures is reversed
- *
- * @param measureRawColumnChunk
- */
- private void updateMeasureRawColumnChunkMinMaxValues(
- MeasureRawColumnChunk measureRawColumnChunk) {
- if (blockInfos.get(index).isDataBlockFromOldStore()) {
- byte[][] maxValues = measureRawColumnChunk.getMaxValues();
- byte[][] minValues = measureRawColumnChunk.getMinValues();
- measureRawColumnChunk.setMaxValues(minValues);
- measureRawColumnChunk.setMinValues(maxValues);
- }
- }
-
- private void updateMeasureRawColumnChunkMinMaxValues(
- MeasureRawColumnChunk[] measureRawColumnChunks) {
- if (blockInfos.get(index).isDataBlockFromOldStore()) {
- for (int i = 0; i < measureRawColumnChunks.length; i++) {
- if (null != measureRawColumnChunks[i]) {
- updateMeasureRawColumnChunkMinMaxValues(measureRawColumnChunks[i]);
- }
- }
- }
- }
-
- private DimensionColumnChunkReader getDimensionColumnChunkReader(FileHolder fileReader) {
- ColumnarFormatVersion version =
- ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- if (fileReader.isReadPageByPage()) {
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
- blockInfos.get(index).getFilePath(), true);
- } else {
- return CarbonDataReaderFactory.getInstance().getDimensionColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
- blockInfos.get(index).getFilePath(), false);
- }
- }
-
- private MeasureColumnChunkReader getMeasureColumnChunkReader(FileHolder fileReader) {
- ColumnarFormatVersion version =
- ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
- if (fileReader.isReadPageByPage()) {
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath(), true);
- } else {
- return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
- blockInfos.get(index).getDetailInfo().getBlockletInfo(),
- blockInfos.get(index).getFilePath(), false);
- }
- }
-
- @Override
- public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
- this.deleteDeltaDataCache = deleteDeltaDataCache;
- }
-
- @Override public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return deleteDeltaDataCache;
- }
-
- @Override public int numberOfPages() {
- return blockInfos.get(index).getDetailInfo().getPagesCount();
- }
-
- @Override public int getPageRowCount(int pageNumber) {
- return blockInfos.get(index).getDetailInfo().getBlockletInfo()
- .getNumberOfRowsPerPage()[pageNumber];
- }
-
- public int numberOfNodes() {
- return blockInfos.size();
- }
-
- public List<TableBlockInfo> getBlockInfos() {
- return blockInfos;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
index 17ad17f..a30f64c 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -32,7 +32,7 @@ public class IndexWrapper extends AbstractIndex {
public IndexWrapper(List<TableBlockInfo> blockInfos) {
segmentProperties = new SegmentProperties(blockInfos.get(0).getDetailInfo().getColumnSchemas(),
blockInfos.get(0).getDetailInfo().getDimLens());
- dataRefNode = new BlockletDataRefNodeWrapper(blockInfos, 0,
+ dataRefNode = new BlockletDataRefNode(blockInfos, 0,
segmentProperties.getDimensionColumnsValueSize());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index 9364a7a..6803fc8 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -19,7 +19,12 @@ package org.apache.carbondata.core.indexstore.blockletindex;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index 242995b..53cbb1d 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -17,11 +17,11 @@
package org.apache.carbondata.core.memory;
+import javax.annotation.concurrent.GuardedBy;
import java.lang.ref.WeakReference;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-import javax.annotation.concurrent.GuardedBy;
import org.apache.carbondata.core.util.CarbonProperties;
[35/54] [abbrv] carbondata git commit: [CARBONDATA-2091][DataLoad]
Support specifying sort column bounds in data loading
Posted by ra...@apache.org.
[CARBONDATA-2091][DataLoad] Support specifying sort column bounds in data loading
Enhance data loading performance by specifying sort column bounds
1. Add row range number during convert-process-step
2. Dispatch rows to each sorter by range number
3. Sort/Write process step can be done concurrently in each range
4. Since all sorttemp files will be written in one folders, we add range
number to the file name to distingush them
Tests added and docs updated
This closes #1953
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d5396b15
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d5396b15
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d5396b15
Branch: refs/heads/master
Commit: d5396b154433dc3e24e7b95666375f7f2981fea1
Parents: e72bfd1
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Tue Feb 13 10:58:06 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../constants/CarbonLoadOptionConstants.java | 10 +
.../core/datastore/row/CarbonRow.java | 10 +-
.../ThriftWrapperSchemaConverterImpl.java | 2 +-
.../core/metadata/schema/BucketingInfo.java | 24 +-
.../core/metadata/schema/ColumnRangeInfo.java | 29 ++
.../metadata/schema/SortColumnRangeInfo.java | 83 +++++
docs/data-management-on-carbondata.md | 11 +
.../TestLoadDataWithSortColumnBounds.scala | 348 +++++++++++++++++++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../carbondata/spark/rdd/PartitionDropper.scala | 2 +-
.../spark/rdd/PartitionSplitter.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../strategy/CarbonLateDecodeStrategy.scala | 2 +-
.../loading/CarbonDataLoadConfiguration.java | 11 +
.../loading/DataLoadProcessBuilder.java | 77 +++-
.../loading/converter/RowConverter.java | 2 +-
.../converter/impl/RowConverterImpl.java | 5 +
.../loading/model/CarbonLoadModel.java | 14 +
.../loading/model/CarbonLoadModelBuilder.java | 1 +
.../processing/loading/model/LoadOption.java | 1 +
.../partition/impl/HashPartitionerImpl.java | 10 +-
.../partition/impl/RangePartitionerImpl.java | 71 ++++
.../partition/impl/RawRowComparator.java | 63 ++++
.../processing/loading/sort/SorterFactory.java | 16 +-
...arallelReadMergeSorterWithBucketingImpl.java | 272 ---------------
...allelReadMergeSorterWithColumnRangeImpl.java | 289 +++++++++++++++
...arallelReadMergeSorterWithBucketingImpl.java | 263 --------------
...allelReadMergeSorterWithColumnRangeImpl.java | 293 ++++++++++++++++
.../loading/sort/unsafe/UnsafeSortDataRows.java | 6 +-
.../unsafe/merger/UnsafeIntermediateMerger.java | 6 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 11 +-
.../steps/DataConverterProcessorStepImpl.java | 102 +++++-
...ConverterProcessorWithBucketingStepImpl.java | 161 ---------
.../steps/DataWriterProcessorStepImpl.java | 70 +++-
.../SingleThreadFinalSortFilesMerger.java | 3 +-
.../processing/sort/sortdata/SortDataRows.java | 11 +-
.../sortdata/SortIntermediateFileMerger.java | 6 +-
.../sort/sortdata/SortParameters.java | 10 +
.../store/CarbonFactDataHandlerColumnar.java | 6 +-
39 files changed, 1558 insertions(+), 750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a6bf60f..8ff8dc4 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -124,4 +124,14 @@ public final class CarbonLoadOptionConstants {
public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION
= "carbon.load.skewedDataOptimization.enabled";
public static final String ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT = "false";
+
+ /**
+ * field delimiter for each field in one bound
+ */
+ public static final String SORT_COLUMN_BOUNDS_FIELD_DELIMITER = ",";
+
+ /**
+ * row delimiter for each sort column bounds
+ */
+ public static final String SORT_COLUMN_BOUNDS_ROW_DELIMITER = ";";
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
index 8702421..bb624af 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
@@ -29,7 +29,7 @@ public class CarbonRow implements Serializable {
private Object[] rawData;
- public short bucketNumber;
+ private short rangeId;
public CarbonRow(Object[] data) {
this.data = data;
@@ -83,4 +83,12 @@ public class CarbonRow implements Serializable {
public void setRawData(Object[] rawData) {
this.rawData = rawData;
}
+
+ public short getRangeId() {
+ return rangeId;
+ }
+
+ public void setRangeId(short rangeId) {
+ this.rangeId = rangeId;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 21ab797..a15804e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -293,7 +293,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.add(fromWrapperToExternalColumnSchema(wrapperColumnSchema));
}
return new org.apache.carbondata.format.BucketingInfo(thriftColumnSchema,
- bucketingInfo.getNumberOfBuckets());
+ bucketingInfo.getNumOfRanges());
}
/* (non-Javadoc)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java
index 569241d..e24f0f8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/BucketingInfo.java
@@ -24,40 +24,41 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
/**
* Bucketing information
*/
-public class BucketingInfo implements Serializable, Writable {
-
+@InterfaceAudience.Internal
+public class BucketingInfo implements ColumnRangeInfo, Serializable, Writable {
private static final long serialVersionUID = -0L;
-
private List<ColumnSchema> listOfColumns;
-
- private int numberOfBuckets;
+ // number of value ranges
+ private int numOfRanges;
public BucketingInfo() {
}
- public BucketingInfo(List<ColumnSchema> listOfColumns, int numberOfBuckets) {
+ public BucketingInfo(List<ColumnSchema> listOfColumns, int numberOfRanges) {
this.listOfColumns = listOfColumns;
- this.numberOfBuckets = numberOfBuckets;
+ this.numOfRanges = numberOfRanges;
}
public List<ColumnSchema> getListOfColumns() {
return listOfColumns;
}
- public int getNumberOfBuckets() {
- return numberOfBuckets;
+ @Override
+ public int getNumOfRanges() {
+ return numOfRanges;
}
@Override
public void write(DataOutput output) throws IOException {
- output.writeInt(numberOfBuckets);
+ output.writeInt(numOfRanges);
output.writeInt(listOfColumns.size());
for (ColumnSchema aColSchema : listOfColumns) {
aColSchema.write(output);
@@ -66,7 +67,7 @@ public class BucketingInfo implements Serializable, Writable {
@Override
public void readFields(DataInput input) throws IOException {
- this.numberOfBuckets = input.readInt();
+ this.numOfRanges = input.readInt();
int colSchemaSize = input.readInt();
this.listOfColumns = new ArrayList<>(colSchemaSize);
for (int i = 0; i < colSchemaSize; i++) {
@@ -75,5 +76,4 @@ public class BucketingInfo implements Serializable, Writable {
this.listOfColumns.add(aSchema);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/metadata/schema/ColumnRangeInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/ColumnRangeInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/ColumnRangeInfo.java
new file mode 100644
index 0000000..c5454b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/ColumnRangeInfo.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+/**
+ * interface for column range information. Currently we treat bucket and sort_column_range as
+ * value ranges for a column.
+ */
+@InterfaceAudience.Internal
+public interface ColumnRangeInfo {
+ int getNumOfRanges();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/core/src/main/java/org/apache/carbondata/core/metadata/schema/SortColumnRangeInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/SortColumnRangeInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SortColumnRangeInfo.java
new file mode 100644
index 0000000..9d2460a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SortColumnRangeInfo.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+/**
+ * column ranges specified by sort column bounds
+ */
+@InterfaceAudience.Internal
+public class SortColumnRangeInfo implements ColumnRangeInfo, Serializable {
+ private static final long serialVersionUID = 1L;
+ // indices for the sort columns in the raw row
+ private int[] sortColumnIndex;
+ // is the sort column no dictionary encoded
+ private boolean[] isSortColumnNoDict;
+ // each literal sort column bounds specified by user
+ private String[] userSpecifiedRanges;
+ // separator for the field values in each bound
+ private String separator;
+ // number of value ranges for the columns
+ private int numOfRanges;
+
+ public SortColumnRangeInfo(int[] sortColumnIndex, boolean[] isSortColumnNoDict,
+ String[] userSpecifiedRanges, String separator) {
+ this.sortColumnIndex = sortColumnIndex;
+ this.isSortColumnNoDict = isSortColumnNoDict;
+ this.userSpecifiedRanges = userSpecifiedRanges;
+ this.separator = separator;
+ this.numOfRanges = userSpecifiedRanges.length + 1;
+ }
+
+ public int[] getSortColumnIndex() {
+ return sortColumnIndex;
+ }
+
+ public boolean[] getIsSortColumnNoDict() {
+ return isSortColumnNoDict;
+ }
+
+ public String[] getUserSpecifiedRanges() {
+ return userSpecifiedRanges;
+ }
+
+ public String getSeparator() {
+ return separator;
+ }
+
+ @Override
+ public int getNumOfRanges() {
+ return numOfRanges;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("SortColumnRangeInfo{");
+ sb.append("sortColumnIndex=").append(Arrays.toString(sortColumnIndex));
+ sb.append(", isSortColumnNoDict=").append(Arrays.toString(isSortColumnNoDict));
+ sb.append(", userSpecifiedRanges=").append(Arrays.toString(userSpecifiedRanges));
+ sb.append(", separator='").append(separator).append('\'');
+ sb.append(", numOfRanges=").append(numOfRanges);
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 2aa4a49..cb9a17c 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -418,6 +418,17 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
```
NOTE: Date formats are specified by date pattern strings. The date pattern letters in CarbonData are same as in JAVA. Refer to [SimpleDateFormat](http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html).
+ - **SORT COLUMN BOUNDS:** Range bounds for sort columns.
+
+ ```
+ OPTIONS('SORT_COLUMN_BOUNDS'='v11,v21,v31;v12,v22,v32;v13,v23,v33')
+ ```
+ NOTE:
+ * SORT_COLUMN_BOUNDS will be used only when the SORT_SCOPE is 'local_sort'.
+ * Each bound is separated by ';' and each field value in bound is separated by ','.
+ * Carbondata will use these bounds as ranges to process data concurrently.
+ * Since the actual order and literal order of the dictionary column are not necessarily the same, we do not recommend you to use this feature if the first sort column is 'dictionary_include'.
+
- **SINGLE_PASS:** Single Pass Loading enables single job to finish data loading with dictionary generation on the fly. It enhances performance in the scenarios where the subsequent data loading after initial load involves fewer incremental updates on the dictionary.
This option specifies whether to use single pass for loading data or not. By default this option is set to FALSE.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSortColumnBounds.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSortColumnBounds.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSortColumnBounds.scala
new file mode 100644
index 0000000..1f171b8
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSortColumnBounds.scala
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.dataload
+
+import java.io.{File, FileOutputStream, OutputStreamWriter, Serializable}
+
+import scala.util.Random
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+case class SortColumnBoundRow (id: Int, date: String, country: String, name: String,
+ phoneType: String, serialName: String, salary: Int) extends Serializable
+
+object TestLoadDataWithSortColumnBounds {
+ def generateOneRow(id : Int): SortColumnBoundRow = {
+ SortColumnBoundRow(id,
+ "2015/7/23",
+ s"country$id",
+ s"name$id",
+ s"phone${new Random().nextInt(10000)}",
+ s"ASD${new Random().nextInt(10000)}",
+ 10000 + id)
+ }
+}
+
+class TestLoadDataWithSortColumnBounds extends QueryTest with BeforeAndAfterAll {
+ private val tableName: String = "test_table_with_sort_column_bounds"
+ private val filePath: String = s"$resourcesPath/source_for_sort_column_bounds.csv"
+ private var df: DataFrame = _
+
+ private val dateFormatStr: String = "yyyy/MM/dd"
+ private val totalLineNum = 2000
+
+ private val originDateStatus: String = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
+
+ override def beforeAll(): Unit = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, dateFormatStr)
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ prepareDataFile()
+ prepareDataFrame()
+ }
+
+ override def afterAll(): Unit = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, originDateStatus)
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ new File(filePath).delete()
+ df = null
+ }
+
+ /**
+ * generate loading files based on source.csv but can have more lines
+ */
+ private def prepareDataFile(): Unit = {
+ val file = new File(filePath)
+
+ val sb: StringBuilder = new StringBuilder
+ def generateLine(id : Int): String = {
+ sb.clear()
+ val row = TestLoadDataWithSortColumnBounds.generateOneRow(id)
+ sb.append(row.id).append(',')
+ .append(row.date).append(',')
+ .append(row.country).append(',')
+ .append(row.name).append(',')
+ .append(row.phoneType).append(',')
+ .append(row.serialName).append(',')
+ .append(row.salary)
+ .append(System.lineSeparator())
+ .toString()
+ }
+
+ val outputStream = new FileOutputStream(file)
+ val writer = new OutputStreamWriter(outputStream)
+ for (i <- 1 to totalLineNum) {
+ writer.write(generateLine(i))
+ }
+
+ writer.flush()
+ writer.close()
+ outputStream.flush()
+ outputStream.close()
+ }
+
+ /**
+ * prepare data frame
+ */
+ private def prepareDataFrame(): Unit = {
+ import sqlContext.implicits._
+ df = sqlContext.sparkSession.sparkContext.parallelize(1 to totalLineNum)
+ .map(id => {
+ val row = TestLoadDataWithSortColumnBounds.generateOneRow(id)
+ (row.id, row.date, row.country, row.name, row.phoneType, row.serialName, row.salary)
+ })
+ .toDF("ID", "date", "country", "name", "phoneType", "serialName", "salary")
+ }
+
+ test("load data with sort column bounds: safe mode") {
+ val originStatus = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT, CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name')")
+ // load with 4 bounds
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab1;800,aab1;1200,aab1;1600,aab1')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT, originStatus)
+ }
+
+ test("load data with sort column bounds: unsafe mode") {
+ val originStatus = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT, CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name')")
+ // load with 4 bounds
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab1;800,aab1;1200,aab1;1600,aab1')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.ENABLE_UNSAFE_SORT, originStatus)
+ }
+
+ test("load data with sort column bounds: empty column value in bounds is treated as null") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name')")
+ // bounds have empty value
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='200,aab1;,aab1')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: sort column bounds will be ignored if it is empty.") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name')")
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: number of column value in bounds should match that of sort column") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name')")
+ val e = intercept[Exception] {
+ // number of column value does not match that of sort columns
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab1;800')")
+ }
+
+ assert(e.getMessage.contains(
+ "The number of field in bounds should be equal to that in sort columns." +
+ " Expected 2, actual 1." +
+ " The illegal bound is '800'"))
+
+ val e2 = intercept[Exception] {
+ // number of column value does not match that of sort columns
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab1;800,aab1,def')")
+ }
+
+ assert(e2.getMessage.contains(
+ "The number of field in bounds should be equal to that in sort columns." +
+ " Expected 2, actual 3." +
+ " The illegal bound is '800,aab1,def'"))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: sort column bounds will be ignored if not using local_sort") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata'" +
+ "tblproperties('sort_columns'='ID,name','sort_scope'='global_sort')")
+ // since the sort_scope is 'global_sort', we will ignore the sort column bounds,
+ // so the error in sort_column bounds will not be thrown
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab,extra_field')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: no sort columns explicitly specified" +
+ " means all dimension columns will be sort columns, so bounds should be set correctly") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata'")
+ // the sort_columns will have 5 columns if we don't specify it explicitly
+ val e = intercept[Exception] {
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,aab')")
+ }
+ assert(e.getMessage.contains(
+ "The number of field in bounds should be equal to that in sort columns." +
+ " Expected 5, actual 2." +
+ " The illegal bound is '400,aab'"))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: sort column is global dictionary encoded") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='ID,name','dictionary_include'='ID')")
+ // ID is sort column and dictionary column. Since the actual order and literal order of
+ // this column are not necessarily the same, this will not cause error but will cause data skewed.
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='400,name400;800,name800;1200,name1200;1600,name1600')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data with sort column bounds: sort column is global dictionary encoded" +
+ " but bounds are not in dictionary") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ sql(s"CREATE TABLE $tableName (ID Int, date Timestamp, country String, name String, phonetype String," +
+ "serialname String, salary Int) STORED BY 'carbondata' " +
+ "tblproperties('sort_columns'='name,ID','dictionary_include'='name')")
+ // 'name' is sort column and dictionary column, but value for 'name' in bounds does not exists
+ // in dictionary. It will not cause error but will cause data skewed.
+ sql(s"LOAD DATA INPATH '$filePath' INTO TABLE $tableName " +
+ s" OPTIONS('fileheader'='ID,date,country,name,phonetype,serialname,salary'," +
+ s" 'sort_column_bounds'='nmm400,400;nme800,800;nme1200,1200;nme1600,1600')")
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data frame with sort column bounds") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ df.write
+ .format("carbondata")
+ .option("tableName", tableName)
+ .option("tempCSV", "false")
+ .option("sort_columns", "ID,name")
+ .option("sort_column_bounds", "600,aab1;1200,aab1")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ sql(s"select count(*) from $tableName").show()
+ sql(s"select count(*) from $tableName where ID > 1001").show()
+
+ checkAnswer(sql(s"select count(*) from $tableName"), Row(totalLineNum))
+ checkAnswer(sql(s"select count(*) from $tableName where ID > 1001"), Row(totalLineNum - 1001))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+
+ test("load data frame with sort column bounds: number of column value in bounds should match that of sort column") {
+ sql(s"DROP TABLE IF EXISTS $tableName")
+
+ val e = intercept[Exception] {
+ df.write
+ .format("carbondata")
+ .option("tableName", tableName)
+ .option("tempCSV", "false")
+ .option("sort_columns", "ID,name")
+ .option("sort_column_bounds", "600,aab1;1200,aab1,def")
+ .mode(SaveMode.Overwrite)
+ .save()
+ }
+ assert(e.getMessage.contains(
+ "The number of field in bounds should be equal to that in sort columns." +
+ " Expected 2, actual 3." +
+ " The illegal bound is '1200,aab1,def'"))
+
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index ddb9b32..49a8023 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -163,7 +163,7 @@ class CarbonScanRDD(
var i = 0
val bucketed =
splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).groupBy(f => f.getBucketId)
- (0 until bucketedTable.getNumberOfBuckets).map { bucketId =>
+ (0 until bucketedTable.getNumOfRanges).map { bucketId =>
val bucketPartitions = bucketed.getOrElse(bucketId.toString, Nil)
val multiBlockSplit =
new CarbonMultiBlockSplit(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
index c73065d..3495885 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionDropper.scala
@@ -50,7 +50,7 @@ object PartitionDropper {
val bucketInfo = carbonTable.getBucketingInfo(tableName)
val bucketNumber = bucketInfo match {
case null => 1
- case _ => bucketInfo.getNumberOfBuckets
+ case _ => bucketInfo.getNumOfRanges
}
val partitionIndex = oldPartitionIds.indexOf(Integer.valueOf(partitionId))
val targetPartitionId = partitionInfo.getPartitionType match {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
index 9106cca..0d437f6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/PartitionSplitter.scala
@@ -46,7 +46,7 @@ object PartitionSplitter {
var finalSplitStatus = false
val bucketNumber = bucketInfo match {
case null => 1
- case _ => bucketInfo.getNumberOfBuckets
+ case _ => bucketInfo.getNumOfRanges
}
val partitionInfo = carbonTable.getPartitionInfo(tableName)
val partitioner = PartitionFactory.getPartitioner(partitionInfo)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index b1e4083..54ac7dc 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -881,7 +881,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
- "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", "SKIP_EMPTY_LINE"
+ "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", "SKIP_EMPTY_LINE",
+ "SORT_COLUMN_BOUNDS"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index 668c4cc..f6d5e25 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -437,7 +437,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
val cols = info.getListOfColumns.asScala
val sortColumn = carbonTable.
getDimensionByTableName(carbonTable.getTableName).get(0).getColName
- val numBuckets = info.getNumberOfBuckets
+ val numBuckets = info.getNumOfRanges
val bucketColumns = cols.flatMap { n =>
val attrRef = output.find(_.name.equalsIgnoreCase(n.getColumnName))
attrRef match {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
index 895fb79..c21c57f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/CarbonDataLoadConfiguration.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.SortColumnRangeInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.processing.loading.converter.DictionaryCardinalityFinder;
@@ -107,6 +108,8 @@ public class CarbonDataLoadConfiguration {
*/
private short writingCoresCount;
+ private SortColumnRangeInfo sortColumnRangeInfo;
+
/**
* Flder path to where data should be written for this load.
*/
@@ -366,4 +369,12 @@ public class CarbonDataLoadConfiguration {
public void setDataWritePath(String dataWritePath) {
this.dataWritePath = dataWritePath;
}
+
+ public SortColumnRangeInfo getSortColumnRangeInfo() {
+ return sortColumnRangeInfo;
+ }
+
+ public void setSortColumnRangeInfo(SortColumnRangeInfo sortColumnRangeInfo) {
+ this.sortColumnRangeInfo = sortColumnRangeInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 01b4c33..0ee77da 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -22,23 +22,26 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonLoadOptionConstants;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.schema.SortColumnRangeInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants;
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.loading.sort.SortScopeOptions;
import org.apache.carbondata.processing.loading.steps.CarbonRowDataWriterProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.DataConverterProcessorStepImpl;
-import org.apache.carbondata.processing.loading.steps.DataConverterProcessorWithBucketingStepImpl;
import org.apache.carbondata.processing.loading.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.loading.steps.InputProcessorStepForPartitionImpl;
@@ -52,6 +55,8 @@ import org.apache.commons.lang3.StringUtils;
* It builds the pipe line of steps for loading data to carbon.
*/
public final class DataLoadProcessBuilder {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DataLoadProcessBuilder.class.getName());
public AbstractDataLoadProcessorStep build(CarbonLoadModel loadModel, String[] storeLocation,
CarbonIterator[] inputIterators) throws Exception {
@@ -150,7 +155,7 @@ public final class DataLoadProcessBuilder {
// 2. Converts the data like dictionary or non dictionary or complex objects depends on
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
- new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
+ new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
// 3. Sorts the data by SortColumn or not
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
@@ -252,6 +257,7 @@ public final class DataLoadProcessBuilder {
configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
configuration.setDataWritePath(loadModel.getDataWritePath());
+ setSortColumnInfo(carbonTable, loadModel, configuration);
// For partition loading always use single core as it already runs in multiple
// threads per partition
if (carbonTable.isHivePartitionTable()) {
@@ -262,4 +268,71 @@ public final class DataLoadProcessBuilder {
return configuration;
}
+ /**
+ * set sort column info in configuration
+ * @param carbonTable carbon table
+ * @param loadModel load model
+ * @param configuration configuration
+ */
+ private static void setSortColumnInfo(CarbonTable carbonTable, CarbonLoadModel loadModel,
+ CarbonDataLoadConfiguration configuration) {
+ List<String> sortCols = carbonTable.getSortColumns(carbonTable.getTableName());
+ SortScopeOptions.SortScope sortScope = SortScopeOptions.getSortScope(loadModel.getSortScope());
+ if (!SortScopeOptions.SortScope.LOCAL_SORT.equals(sortScope)
+ || sortCols.size() == 0
+ || StringUtils.isBlank(loadModel.getSortColumnsBoundsStr())) {
+ if (!StringUtils.isBlank(loadModel.getSortColumnsBoundsStr())) {
+ LOGGER.warn("sort column bounds will be ignored");
+ }
+
+ configuration.setSortColumnRangeInfo(null);
+ return;
+ }
+ // column index for sort columns
+ int[] sortColIndex = new int[sortCols.size()];
+ boolean[] isSortColNoDict = new boolean[sortCols.size()];
+
+ DataField[] outFields = configuration.getDataFields();
+ int j = 0;
+ boolean columnExist;
+ for (String sortCol : sortCols) {
+ columnExist = false;
+
+ for (int i = 0; !columnExist && i < outFields.length; i++) {
+ if (outFields[i].getColumn().getColName().equalsIgnoreCase(sortCol)) {
+ columnExist = true;
+
+ sortColIndex[j] = i;
+ isSortColNoDict[j] = !outFields[i].hasDictionaryEncoding();
+ j++;
+ }
+ }
+
+ if (!columnExist) {
+ throw new CarbonDataLoadingException("Field " + sortCol + " does not exist.");
+ }
+ }
+
+ String[] sortColumnBounds = StringUtils.splitPreserveAllTokens(
+ loadModel.getSortColumnsBoundsStr(),
+ CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_ROW_DELIMITER, -1);
+ for (String bound : sortColumnBounds) {
+ String[] fieldInBounds = StringUtils.splitPreserveAllTokens(bound,
+ CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_FIELD_DELIMITER, -1);
+ if (fieldInBounds.length != sortCols.size()) {
+ String msg = new StringBuilder(
+ "The number of field in bounds should be equal to that in sort columns.")
+ .append(" Expected ").append(sortCols.size())
+ .append(", actual ").append(String.valueOf(fieldInBounds.length)).append(".")
+ .append(" The illegal bound is '").append(bound).append("'.").toString();
+ throw new CarbonDataLoadingException(msg);
+ }
+ }
+
+ SortColumnRangeInfo sortColumnRangeInfo = new SortColumnRangeInfo(sortColIndex,
+ isSortColNoDict,
+ sortColumnBounds,
+ CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_FIELD_DELIMITER);
+ configuration.setSortColumnRangeInfo(sortColumnRangeInfo);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/converter/RowConverter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/RowConverter.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/RowConverter.java
index fd3a650..016ff3f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/RowConverter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/RowConverter.java
@@ -31,6 +31,6 @@ public interface RowConverter extends DictionaryCardinalityFinder {
CarbonRow convert(CarbonRow row) throws CarbonDataLoadingException;
RowConverter createCopyForNewThread();
-
+ FieldConverter[] getFieldConverters();
void finish();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
index c5313cb..208d42f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java
@@ -250,4 +250,9 @@ public class RowConverterImpl implements RowConverter {
}
return cardinality;
}
+
+ @Override
+ public FieldConverter[] getFieldConverters() {
+ return fieldConverters;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index bf1dfd1..edaa4f5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -187,6 +187,10 @@ public class CarbonLoadModel implements Serializable {
private String globalSortPartitions;
private boolean isAggLoadRequest;
+ /**
+ * sort columns bounds
+ */
+ private String sortColumnsBoundsStr;
/**
* It directly writes data directly to nosort processor bypassing all other processors.
@@ -365,6 +369,14 @@ public class CarbonLoadModel implements Serializable {
this.dictionaryServiceProvider = dictionaryServiceProvider;
}
+ public String getSortColumnsBoundsStr() {
+ return sortColumnsBoundsStr;
+ }
+
+ public void setSortColumnsBoundsStr(String sortColumnsBoundsStr) {
+ this.sortColumnsBoundsStr = sortColumnsBoundsStr;
+ }
+
/**
* Get copy with taskNo.
* Broadcast value is shared in process, so we need to copy it to make sure the value in each
@@ -415,6 +427,7 @@ public class CarbonLoadModel implements Serializable {
copy.isAggLoadRequest = isAggLoadRequest;
copy.badRecordsLocation = badRecordsLocation;
copy.isPartitionLoad = isPartitionLoad;
+ copy.sortColumnsBoundsStr = sortColumnsBoundsStr;
return copy;
}
@@ -467,6 +480,7 @@ public class CarbonLoadModel implements Serializable {
copyObj.batchSortSizeInMb = batchSortSizeInMb;
copyObj.badRecordsLocation = badRecordsLocation;
copyObj.isAggLoadRequest = isAggLoadRequest;
+ copyObj.sortColumnsBoundsStr = sortColumnsBoundsStr;
return copyObj;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 99684ad..17e8dbe 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -223,6 +223,7 @@ public class CarbonLoadModelBuilder {
carbonLoadModel.setMaxColumns(String.valueOf(validatedMaxColumns));
carbonLoadModel.readAndSetLoadMetadataDetails();
+ carbonLoadModel.setSortColumnsBoundsStr(optionsFinal.get("sort_column_bounds"));
}
private int validateMaxColumns(String[] csvHeaders, String maxColumns)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index bd942ca..5af4859 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -191,6 +191,7 @@ public class LoadOption {
}
optionsFinal.put("single_pass", String.valueOf(singlePass));
+ optionsFinal.put("sort_column_bounds", Maps.getOrDefault(options, "sort_column_bounds", ""));
return optionsFinal;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java
index f24d24f..e10faf6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java
@@ -19,6 +19,8 @@ package org.apache.carbondata.processing.loading.partition.impl;
import java.util.List;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -27,7 +29,8 @@ import org.apache.carbondata.processing.loading.partition.Partitioner;
/**
* Hash partitioner implementation
*/
-public class HashPartitionerImpl implements Partitioner<Object[]> {
+@InterfaceAudience.Internal
+public class HashPartitionerImpl implements Partitioner<CarbonRow> {
private int numberOfBuckets;
@@ -50,10 +53,11 @@ public class HashPartitionerImpl implements Partitioner<Object[]> {
}
}
- @Override public int getPartition(Object[] objects) {
+ @Override
+ public int getPartition(CarbonRow key) {
int hashCode = 0;
for (Hash hash : hashes) {
- hashCode += hash.getHash(objects);
+ hashCode += hash.getHash(key.getData());
}
return (hashCode & Integer.MAX_VALUE) % numberOfBuckets;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RangePartitionerImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RangePartitionerImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RangePartitionerImpl.java
new file mode 100644
index 0000000..d59ad02
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RangePartitionerImpl.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.partition.impl;
+
+import java.util.Arrays;
+import java.util.Comparator;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.processing.loading.partition.Partitioner;
+
+@InterfaceAudience.Internal
+public class RangePartitionerImpl implements Partitioner<CarbonRow> {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(RangePartitionerImpl.class.getName());
+ private CarbonRow[] rangeBounds;
+ private Comparator<CarbonRow> comparator;
+
+ public RangePartitionerImpl(CarbonRow[] rangeBounds, Comparator<CarbonRow> comparator) {
+ this.rangeBounds = rangeBounds;
+ LOGGER.info("Use range partitioner to distribute data to "
+ + (rangeBounds.length + 1) + " ranges.");
+ this.comparator = comparator;
+ }
+
+ /**
+ * learned from spark org.apache.spark.RangePartitioner
+ *
+ * @param key key
+ * @return partitionId
+ */
+ @Override
+ public int getPartition(CarbonRow key) {
+ int partition = 0;
+ if (rangeBounds.length <= 128) {
+ // If we have less than 128 partitions naive search
+ while (partition < rangeBounds.length
+ && comparator.compare(key, rangeBounds[partition]) > 0) {
+ partition += 1;
+ }
+ } else {
+ // binary search. binarySearch either returns the match location or -[insertion point]-1
+ partition = Arrays.binarySearch(rangeBounds, 0, rangeBounds.length, key, comparator);
+ if (partition < 0) {
+ partition = -partition - 1;
+ }
+ if (partition > rangeBounds.length) {
+ partition = rangeBounds.length;
+ }
+ }
+
+ return partition;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java
new file mode 100644
index 0000000..64b64f5
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/RawRowComparator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.partition.impl;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+/**
+ * comparator for the converted row. The row has not been rearranged as 3-parted yet.
+ */
+@InterfaceAudience.Internal
+public class RawRowComparator implements Comparator<CarbonRow> {
+ private int[] sortColumnIndices;
+ private boolean[] isSortColumnNoDict;
+
+ public RawRowComparator(int[] sortColumnIndices, boolean[] isSortColumnNoDict) {
+ this.sortColumnIndices = sortColumnIndices;
+ this.isSortColumnNoDict = isSortColumnNoDict;
+ }
+
+ @Override
+ public int compare(CarbonRow o1, CarbonRow o2) {
+ int diff = 0;
+ int i = 0;
+ for (int colIdx : sortColumnIndices) {
+ if (isSortColumnNoDict[i]) {
+ byte[] colA = (byte[]) o1.getObject(colIdx);
+ byte[] colB = (byte[]) o2.getObject(colIdx);
+ diff = UnsafeComparer.INSTANCE.compareTo(colA, colB);
+ if (diff != 0) {
+ return diff;
+ }
+ } else {
+ int colA = (int) o1.getObject(colIdx);
+ int colB = (int) o2.getObject(colIdx);
+ diff = colA - colB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ i++;
+ }
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
index a8f0282..b74b393 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SorterFactory.java
@@ -25,10 +25,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.loading.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.loading.sort.impl.ParallelReadMergeSorterImpl;
-import org.apache.carbondata.processing.loading.sort.impl.ParallelReadMergeSorterWithBucketingImpl;
+import org.apache.carbondata.processing.loading.sort.impl.ParallelReadMergeSorterWithColumnRangeImpl;
import org.apache.carbondata.processing.loading.sort.impl.UnsafeBatchParallelReadMergeSorterImpl;
import org.apache.carbondata.processing.loading.sort.impl.UnsafeParallelReadMergeSorterImpl;
-import org.apache.carbondata.processing.loading.sort.impl.UnsafeParallelReadMergeSorterWithBucketingImpl;
+import org.apache.carbondata.processing.loading.sort.impl.UnsafeParallelReadMergeSorterWithColumnRangeImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
public class SorterFactory {
@@ -44,15 +44,21 @@ public class SorterFactory {
Sorter sorter;
if (offheapsort) {
if (configuration.getBucketingInfo() != null) {
- sorter = new UnsafeParallelReadMergeSorterWithBucketingImpl(configuration.getDataFields(),
+ sorter = new UnsafeParallelReadMergeSorterWithColumnRangeImpl(counter,
configuration.getBucketingInfo());
+ } else if (configuration.getSortColumnRangeInfo() != null) {
+ sorter = new UnsafeParallelReadMergeSorterWithColumnRangeImpl(counter,
+ configuration.getSortColumnRangeInfo());
} else {
sorter = new UnsafeParallelReadMergeSorterImpl(counter);
}
} else {
if (configuration.getBucketingInfo() != null) {
- sorter =
- new ParallelReadMergeSorterWithBucketingImpl(counter, configuration.getBucketingInfo());
+ sorter = new ParallelReadMergeSorterWithColumnRangeImpl(counter,
+ configuration.getBucketingInfo());
+ } else if (configuration.getSortColumnRangeInfo() != null) {
+ sorter = new ParallelReadMergeSorterWithColumnRangeImpl(counter,
+ configuration.getSortColumnRangeInfo());
} else {
sorter = new ParallelReadMergeSorterImpl(counter);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
deleted file mode 100644
index b7452a7..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.loading.sort.impl;
-
-import java.io.File;
-import java.util.Iterator;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.metadata.schema.BucketingInfo;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
-import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
-import org.apache.carbondata.processing.loading.sort.AbstractMergeSorter;
-import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
-import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
-import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-
-/**
- * It parallely reads data from array of iterates and do merge sort.
- * First it sorts the data and write to temp files. These temp files will be merge sorted to get
- * final merge sort result.
- * This step is specifically for bucketing, it sorts each bucket data separately and write to
- * temp files.
- */
-public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ParallelReadMergeSorterWithBucketingImpl.class.getName());
-
- private SortParameters sortParameters;
-
- private SortIntermediateFileMerger[] intermediateFileMergers;
-
- private BucketingInfo bucketingInfo;
-
- private int sortBufferSize;
-
- private AtomicLong rowCounter;
-
- public ParallelReadMergeSorterWithBucketingImpl(AtomicLong rowCounter,
- BucketingInfo bucketingInfo) {
- this.rowCounter = rowCounter;
- this.bucketingInfo = bucketingInfo;
- }
-
- @Override public void initialize(SortParameters sortParameters) {
- this.sortParameters = sortParameters;
- int buffer = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL));
- sortBufferSize = buffer / bucketingInfo.getNumberOfBuckets();
- if (sortBufferSize < 100) {
- sortBufferSize = 100;
- }
- }
-
- @Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators)
- throws CarbonDataLoadingException {
- SortDataRows[] sortDataRows = new SortDataRows[bucketingInfo.getNumberOfBuckets()];
- intermediateFileMergers =
- new SortIntermediateFileMerger[sortDataRows.length];
- try {
- for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
- SortParameters parameters = sortParameters.getCopy();
- parameters.setPartitionID(i + "");
- setTempLocation(parameters);
- parameters.setBufferSize(sortBufferSize);
- intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
- sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
- sortDataRows[i].initialize();
- }
- } catch (CarbonSortKeyAndGroupByException e) {
- throw new CarbonDataLoadingException(e);
- }
- ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
- this.threadStatusObserver = new ThreadStatusObserver(executorService);
- final int batchSize = CarbonProperties.getInstance().getBatchSize();
- try {
- for (int i = 0; i < iterators.length; i++) {
- executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter,
- this.threadStatusObserver));
- }
- executorService.shutdown();
- executorService.awaitTermination(2, TimeUnit.DAYS);
- processRowToNextStep(sortDataRows, sortParameters);
- } catch (Exception e) {
- checkError();
- throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
- }
- checkError();
- try {
- for (int i = 0; i < intermediateFileMergers.length; i++) {
- intermediateFileMergers[i].finish();
- }
- } catch (CarbonDataWriterException e) {
- throw new CarbonDataLoadingException(e);
- } catch (CarbonSortKeyAndGroupByException e) {
- throw new CarbonDataLoadingException(e);
- }
-
- Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
- for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
- batchIterator[i] = new MergedDataIterator(String.valueOf(i), batchSize);
- }
-
- return batchIterator;
- }
-
- private SingleThreadFinalSortFilesMerger getFinalMerger(String bucketId) {
- String[] storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(
- sortParameters.getDatabaseName(), sortParameters.getTableName(),
- String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(),
- false, false);
- // Set the data file location
- String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation, File.separator,
- CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
- return new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
- sortParameters);
- }
-
- @Override public void close() {
- for (int i = 0; i < intermediateFileMergers.length; i++) {
- intermediateFileMergers[i].close();
- }
- }
-
- /**
- * Below method will be used to process data to next step
- */
- private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters)
- throws CarbonDataLoadingException {
- if (null == sortDataRows || sortDataRows.length == 0) {
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- LOGGER.info("Number of Records was Zero");
- String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
- LOGGER.info(logMessage);
- return false;
- }
-
- try {
- for (int i = 0; i < sortDataRows.length; i++) {
- // start sorting
- sortDataRows[i].startSorting();
- }
- // check any more rows are present
- LOGGER.info("Record Processed For table: " + parameters.getTableName());
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
- .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
- return false;
- } catch (CarbonSortKeyAndGroupByException e) {
- throw new CarbonDataLoadingException(e);
- }
- }
-
- private void setTempLocation(SortParameters parameters) {
- String[] carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(
- parameters.getDatabaseName(), parameters.getTableName(), parameters.getTaskNo(),
- parameters.getSegmentId(), false, false);
- String[] tmpLocs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
- CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
- parameters.setTempFileLocation(tmpLocs);
- }
-
- /**
- * This thread iterates the iterator and adds the rows to @{@link SortDataRows}
- */
- private static class SortIteratorThread implements Runnable {
-
- private Iterator<CarbonRowBatch> iterator;
-
- private SortDataRows[] sortDataRows;
-
- private AtomicLong rowCounter;
-
- private ThreadStatusObserver threadStatusObserver;
-
- public SortIteratorThread(Iterator<CarbonRowBatch> iterator, SortDataRows[] sortDataRows,
- AtomicLong rowCounter, ThreadStatusObserver observer) {
- this.iterator = iterator;
- this.sortDataRows = sortDataRows;
- this.rowCounter = rowCounter;
- this.threadStatusObserver = observer;
- }
-
- @Override
- public void run() {
- try {
- while (iterator.hasNext()) {
- CarbonRowBatch batch = iterator.next();
- int i = 0;
- while (batch.hasNext()) {
- CarbonRow row = batch.next();
- if (row != null) {
- SortDataRows sortDataRow = sortDataRows[row.bucketNumber];
- synchronized (sortDataRow) {
- sortDataRow.addRow(row.getData());
- rowCounter.getAndAdd(1);
- }
- }
- }
- }
- } catch (Exception e) {
- LOGGER.error(e);
- this.threadStatusObserver.notifyFailed(e);
- }
- }
-
- }
-
- private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
-
- private String partitionId;
-
- private int batchSize;
-
- private boolean firstRow = true;
-
- public MergedDataIterator(String partitionId, int batchSize) {
- this.partitionId = partitionId;
- this.batchSize = batchSize;
- }
-
- private SingleThreadFinalSortFilesMerger finalMerger;
-
- @Override public boolean hasNext() {
- if (firstRow) {
- firstRow = false;
- finalMerger = getFinalMerger(partitionId);
- finalMerger.startFinalMerge();
- }
- return finalMerger.hasNext();
- }
-
- @Override public CarbonRowBatch next() {
- int counter = 0;
- CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
- while (finalMerger.hasNext() && counter < batchSize) {
- rowBatch.addRow(new CarbonRow(finalMerger.next()));
- counter++;
- }
- return rowBatch;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d5396b15/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
new file mode 100644
index 0000000..808952b
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/impl/ParallelReadMergeSorterWithColumnRangeImpl.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.loading.sort.impl;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.schema.ColumnRangeInfo;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.loading.row.CarbonRowBatch;
+import org.apache.carbondata.processing.loading.sort.AbstractMergeSorter;
+import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
+import org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger;
+import org.apache.carbondata.processing.sort.sortdata.SortDataRows;
+import org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger;
+import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+/**
+ * It parallely reads data from array of iterates and do merge sort.
+ * First it sorts the data and write to temp files. These temp files will be merge sorted to get
+ * final merge sort result.
+ * This step is specifically for the data loading with specifying column value range, such as
+ * bucketing,sort_column_bounds, it sorts each range of data separately and write to temp files.
+ */
+public class ParallelReadMergeSorterWithColumnRangeImpl extends AbstractMergeSorter {
+ private static final LogService LOGGER = LogServiceFactory.getLogService(
+ ParallelReadMergeSorterWithColumnRangeImpl.class.getName());
+
+ private SortParameters originSortParameters;
+
+ private SortIntermediateFileMerger[] intermediateFileMergers;
+
+ private ColumnRangeInfo columnRangeInfo;
+
+ private int sortBufferSize;
+
+ private AtomicLong rowCounter;
+ /**
+ * counters to collect information about rows processed by each range
+ */
+ private List<AtomicLong> insideRowCounterList;
+
+ public ParallelReadMergeSorterWithColumnRangeImpl(AtomicLong rowCounter,
+ ColumnRangeInfo columnRangeInfo) {
+ this.rowCounter = rowCounter;
+ this.columnRangeInfo = columnRangeInfo;
+ }
+
+ @Override
+ public void initialize(SortParameters sortParameters) {
+ this.originSortParameters = sortParameters;
+ int buffer = Integer.parseInt(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL));
+ sortBufferSize = buffer / columnRangeInfo.getNumOfRanges();
+ if (sortBufferSize < 100) {
+ sortBufferSize = 100;
+ }
+ this.insideRowCounterList = new ArrayList<>(columnRangeInfo.getNumOfRanges());
+ for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
+ insideRowCounterList.add(new AtomicLong(0));
+ }
+ }
+
+ @Override public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators)
+ throws CarbonDataLoadingException {
+ SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
+ intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
+ SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
+ try {
+ for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
+ SortParameters parameters = originSortParameters.getCopy();
+ parameters.setPartitionID(i + "");
+ parameters.setRangeId(i);
+ sortParameterArray[i] = parameters;
+ setTempLocation(parameters);
+ parameters.setBufferSize(sortBufferSize);
+ intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
+ sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
+ sortDataRows[i].initialize();
+ }
+ } catch (CarbonSortKeyAndGroupByException e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
+ this.threadStatusObserver = new ThreadStatusObserver(executorService);
+ final int batchSize = CarbonProperties.getInstance().getBatchSize();
+ try {
+ // dispatch rows to sortDataRows by range id
+ for (int i = 0; i < iterators.length; i++) {
+ executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter,
+ this.insideRowCounterList, this.threadStatusObserver));
+ }
+ executorService.shutdown();
+ executorService.awaitTermination(2, TimeUnit.DAYS);
+ processRowToNextStep(sortDataRows, originSortParameters);
+ } catch (Exception e) {
+ checkError();
+ throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
+ }
+ checkError();
+ try {
+ for (int i = 0; i < intermediateFileMergers.length; i++) {
+ intermediateFileMergers[i].finish();
+ }
+ } catch (CarbonDataWriterException e) {
+ throw new CarbonDataLoadingException(e);
+ } catch (CarbonSortKeyAndGroupByException e) {
+ throw new CarbonDataLoadingException(e);
+ }
+
+ Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
+ for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
+ batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
+ }
+
+ return batchIterator;
+ }
+
+ private SingleThreadFinalSortFilesMerger getFinalMerger(SortParameters sortParameters) {
+ String[] storeLocation = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(),
+ String.valueOf(sortParameters.getTaskNo()),
+ sortParameters.getSegmentId() + "", false, false);
+ // Set the data file location
+ String[] dataFolderLocation = CarbonDataProcessorUtil.arrayAppend(storeLocation, File.separator,
+ CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
+ return new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
+ sortParameters);
+ }
+
+ @Override public void close() {
+ for (int i = 0; i < intermediateFileMergers.length; i++) {
+ intermediateFileMergers[i].close();
+ }
+ }
+
+ /**
+ * Below method will be used to process data to next step
+ */
+ private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters)
+ throws CarbonDataLoadingException {
+ if (null == sortDataRows || sortDataRows.length == 0) {
+ LOGGER.info("Record Processed For table: " + parameters.getTableName());
+ LOGGER.info("Number of Records was Zero");
+ String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
+ LOGGER.info(logMessage);
+ return false;
+ }
+
+ try {
+ for (int i = 0; i < sortDataRows.length; i++) {
+ // start sorting
+ sortDataRows[i].startSorting();
+ }
+ // check any more rows are present
+ LOGGER.info("Record Processed For table: " + parameters.getTableName());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
+ return false;
+ } catch (CarbonSortKeyAndGroupByException e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ }
+
+ private void setTempLocation(SortParameters parameters) {
+ String[] carbonDataDirectoryPath = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(parameters.getDatabaseName(),
+ parameters.getTableName(), parameters.getTaskNo(),
+ parameters.getSegmentId(), false, false);
+ String[] tmpLocs = CarbonDataProcessorUtil.arrayAppend(carbonDataDirectoryPath, File.separator,
+ CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
+ parameters.setTempFileLocation(tmpLocs);
+ }
+
+ /**
+ * This thread iterates the iterator and adds the rows to @{@link SortDataRows}
+ */
+ private static class SortIteratorThread implements Runnable {
+
+ private Iterator<CarbonRowBatch> iterator;
+
+ private SortDataRows[] sortDataRows;
+
+ private AtomicLong rowCounter;
+ private List<AtomicLong> insideCounterList;
+ private ThreadStatusObserver threadStatusObserver;
+
+ public SortIteratorThread(Iterator<CarbonRowBatch> iterator, SortDataRows[] sortDataRows,
+ AtomicLong rowCounter, List<AtomicLong> insideCounterList,
+ ThreadStatusObserver observer) {
+ this.iterator = iterator;
+ this.sortDataRows = sortDataRows;
+ this.rowCounter = rowCounter;
+ this.insideCounterList = insideCounterList;
+ this.threadStatusObserver = observer;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (iterator.hasNext()) {
+ CarbonRowBatch batch = iterator.next();
+ while (batch.hasNext()) {
+ CarbonRow row = batch.next();
+ if (row != null) {
+ SortDataRows sortDataRow = sortDataRows[row.getRangeId()];
+ synchronized (sortDataRow) {
+ sortDataRow.addRow(row.getData());
+ insideCounterList.get(row.getRangeId()).getAndIncrement();
+ rowCounter.getAndAdd(1);
+ }
+ }
+ }
+ }
+ LOGGER.info("Rows processed by each range: " + insideCounterList);
+ } catch (Exception e) {
+ LOGGER.error(e);
+ this.threadStatusObserver.notifyFailed(e);
+ }
+ }
+
+ }
+
+ private class MergedDataIterator extends CarbonIterator<CarbonRowBatch> {
+
+ private SortParameters sortParameters;
+
+ private int batchSize;
+
+ private boolean firstRow = true;
+
+ public MergedDataIterator(SortParameters sortParameters, int batchSize) {
+ this.sortParameters = sortParameters;
+ this.batchSize = batchSize;
+ }
+
+ private SingleThreadFinalSortFilesMerger finalMerger;
+
+ @Override public boolean hasNext() {
+ if (firstRow) {
+ firstRow = false;
+ finalMerger = getFinalMerger(sortParameters);
+ finalMerger.startFinalMerge();
+ }
+ return finalMerger.hasNext();
+ }
+
+ @Override public CarbonRowBatch next() {
+ int counter = 0;
+ CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
+ while (finalMerger.hasNext() && counter < batchSize) {
+ rowBatch.addRow(new CarbonRow(finalMerger.next()));
+ counter++;
+ }
+ return rowBatch;
+ }
+ }
+}
[38/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
new file mode 100644
index 0000000..795ef6a
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/IndexDataMapWriterSuite.scala
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.{DataFrame, SaveMode}
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter}
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainIndexDataMap, AbstractCoarseGrainIndexDataMapFactory}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.events.Event
+
+class C2IndexDataMapFactory() extends AbstractCoarseGrainIndexDataMapFactory {
+
+ var identifier: AbsoluteTableIdentifier = _
+
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ }
+
+ override def fireEvent(event: Event): Unit = ???
+
+ override def clear(segment: Segment): Unit = {}
+
+ override def clear(): Unit = {}
+
+ override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainIndexDataMap] = ???
+
+ override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainIndexDataMap] = ???
+
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter =
+ IndexDataMapWriterSuite.dataMapWriterC2Mock(identifier, segment, dataWritePath)
+
+ override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = {
+ ???
+ }
+
+}
+
+class IndexDataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
+ def buildTestData(numRows: Int): DataFrame = {
+ import sqlContext.implicits._
+ sqlContext.sparkContext.parallelize(1 to numRows, 1)
+ .map(x => ("a" + x, "b", x))
+ .toDF("c1", "c2", "c3")
+ }
+
+ def dropTable(): Unit = {
+ sql("DROP TABLE IF EXISTS carbon1")
+ sql("DROP TABLE IF EXISTS carbon2")
+ }
+
+ override def beforeAll {
+ dropTable()
+ }
+
+ test("test write datamap 2 pages") {
+ sql(s"CREATE TABLE carbon1(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
+ // register datamap writer
+ sql(s"CREATE DATAMAP test ON TABLE carbon1 USING '${classOf[C2IndexDataMapFactory].getName}'")
+ val df = buildTestData(33000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon1")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(IndexDataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(IndexDataMapWriterSuite.callbackSeq.last.contains("block end"))
+ assert(
+ IndexDataMapWriterSuite.callbackSeq.slice(1, IndexDataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "blocklet end: 0"
+ ))
+ IndexDataMapWriterSuite.callbackSeq = Seq()
+ }
+
+ test("test write datamap 2 blocklet") {
+ sql(s"CREATE TABLE carbon2(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
+ sql(s"CREATE DATAMAP test ON TABLE carbon2 USING '${classOf[C2IndexDataMapFactory].getName}'")
+
+ CarbonProperties.getInstance()
+ .addProperty("carbon.blockletgroup.size.in.mb", "1")
+ CarbonProperties.getInstance()
+ .addProperty("carbon.number.of.cores.while.loading",
+ CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)
+
+ val df = buildTestData(300000)
+
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon2")
+ .option("tempCSV", "false")
+ .option("sort_columns","c1")
+ .option("SORT_SCOPE","GLOBAL_SORT")
+ .mode(SaveMode.Overwrite)
+ .save()
+
+ assert(IndexDataMapWriterSuite.callbackSeq.head.contains("block start"))
+ assert(IndexDataMapWriterSuite.callbackSeq.last.contains("block end"))
+ // corrected test case the min "carbon.blockletgroup.size.in.mb" size could not be less than
+ // 64 MB
+ assert(
+ IndexDataMapWriterSuite.callbackSeq.slice(1, IndexDataMapWriterSuite.callbackSeq.length - 1) == Seq(
+ "blocklet start 0",
+ "add page data: blocklet 0, page 0",
+ "add page data: blocklet 0, page 1",
+ "add page data: blocklet 0, page 2",
+ "add page data: blocklet 0, page 3",
+ "add page data: blocklet 0, page 4",
+ "add page data: blocklet 0, page 5",
+ "add page data: blocklet 0, page 6",
+ "add page data: blocklet 0, page 7",
+ "add page data: blocklet 0, page 8",
+ "add page data: blocklet 0, page 9",
+ "blocklet end: 0"
+ ))
+ IndexDataMapWriterSuite.callbackSeq = Seq()
+ }
+
+ override def afterAll {
+ dropTable()
+ }
+}
+
+object IndexDataMapWriterSuite {
+
+ var callbackSeq: Seq[String] = Seq[String]()
+
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segment: Segment,
+ dataWritePath: String) =
+ new AbstractDataMapWriter(identifier, segment, dataWritePath) {
+
+ override def onPageAdded(
+ blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ assert(pages.length == 1)
+ assert(pages(0).getDataType == DataTypes.STRING)
+ val bytes: Array[Byte] = pages(0).getByteArrayPage()(0)
+ assert(bytes.sameElements(Seq(0, 1, 'b'.toByte)))
+ callbackSeq :+= s"add page data: blocklet $blockletId, page $pageId"
+ }
+
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ callbackSeq :+= s"blocklet end: $blockletId"
+ }
+
+ override def onBlockEnd(blockId: String): Unit = {
+ callbackSeq :+= s"block end $blockId"
+ }
+
+ override def onBlockletStart(blockletId: Int): Unit = {
+ callbackSeq :+= s"blocklet start $blockletId"
+ }
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String) = {
+ callbackSeq :+= s"block start $blockId"
+ }
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish() = {
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
deleted file mode 100644
index 717af6f..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{File, FilenameFilter}
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.common.exceptions.MetadataProcessException
-import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonTablePath
-
-class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
-
- val testData = s"$resourcesPath/sample.csv"
-
- override def beforeAll {
- sql("drop table if exists datamaptest")
- sql("drop table if exists datamapshowtest")
- sql("drop table if exists uniqdata")
- sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
- }
-
- val newClass = "org.apache.spark.sql.CarbonSource"
-
- test("test datamap create: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
- }
- }
-
- test("test datamap create with dmproperties: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- }
- }
-
- test("test datamap create with existing name: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(
- s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- }
- }
-
- test("test datamap create with preagg") {
- sql("drop datamap if exists datamap3 on table datamaptest")
- sql(
- "create datamap datamap3 on table datamaptest using 'preaggregate' dmproperties('key'='value') as select count(a) from datamaptest")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 1)
- assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
- assert(dataMapSchemaList.get(0).getProperties.get("key").equals("value"))
- assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
- }
-
- test("check hivemetastore after drop datamap") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- "true")
- sql("drop table if exists hiveMetaStoreTable")
- sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
-
- sql(
- "create datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable using 'preaggregate' dmproperties('key'='value') as select count(a) from hiveMetaStoreTable")
- checkExistence(sql("show datamap on table hiveMetaStoreTable"), true, "datamap_hiveMetaStoreTable")
-
- sql("drop datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
- checkExistence(sql("show datamap on table hiveMetaStoreTable"), false, "datamap_hiveMetaStoreTable")
-
- } finally {
- sql("drop table hiveMetaStoreTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("drop the table having pre-aggregate") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- "true")
- sql("drop table if exists hiveMetaStoreTable_1")
- sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
-
- sql(
- "create datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1 using 'preaggregate' dmproperties('key'='value') as select count(a) from hiveMetaStoreTable_1")
-
- checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
- true,
- "datamap_hiveMetaStoreTable_1")
-
- sql("drop datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
- checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
- false,
- "datamap_hiveMetaStoreTable_1")
- assert(sql("show datamap on table hiveMetaStoreTable_1").collect().length == 0)
- sql("drop table hiveMetaStoreTable_1")
-
- checkExistence(sql("show tables"), false, "datamap_hiveMetaStoreTable_1")
- }
- finally {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("test datamap create with preagg with duplicate name") {
- sql(
- s"""
- | CREATE DATAMAP datamap10 ON TABLE datamaptest
- | USING 'preaggregate'
- | DMPROPERTIES('key'='value')
- | AS SELECT COUNT(a) FROM datamaptest
- """.stripMargin)
- intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP datamap10 ON TABLE datamaptest
- | USING 'preaggregate'
- | DMPROPERTIES('key'='value')
- | AS SELECT COUNT(a) FROM datamaptest
- """.stripMargin)
- }
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- }
-
- test("test drop non-exist datamap") {
- intercept[NoSuchDataMapException] {
- sql("drop datamap nonexist on table datamaptest")
- }
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- }
-
- test("test show datamap without preaggregate: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
- checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
- }
- }
-
- test("test show datamap with preaggregate: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 2)
- checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
- }
- }
-
- test("test show datamap with no datamap") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- assert(sql("show datamap on table datamapshowtest").collect().length == 0)
- }
-
- test("test show datamap after dropping datamap: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
- sql("drop datamap datamap1 on table datamapshowtest")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 1)
- checkExistence(frame, true, "datamap2", "(NA)", newClass)
- }
- }
-
- test("test if preaggregate load is successfull for hivemetastore") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
-
- .stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- } finally {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("test preaggregate load for decimal column for hivemetastore") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
- sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
- sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
- checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
- sql("drop datamap if exists uniqdata_agg on table uniqdata")
- }
-
- test("create pre-agg table with path") {
- sql("drop table if exists main_preagg")
- sql("drop table if exists main ")
- val warehouse = s"$metastoredb/warehouse"
- val path = warehouse + "/" + System.nanoTime + "_preAggTestPath"
- sql(
- s"""
- | create table main(
- | year int,
- | month int,
- | name string,
- | salary int)
- | stored by 'carbondata'
- | tblproperties('sort_columns'='month,year,name')
- """.stripMargin)
- sql("insert into main select 10,11,'amy',12")
- sql("insert into main select 10,11,'amy',14")
- sql(
- s"""
- | create datamap preagg
- | on table main
- | using 'preaggregate'
- | dmproperties ('path'='$path')
- | as select name,avg(salary)
- | from main
- | group by name
- """.stripMargin)
- assertResult(true)(new File(path).exists())
- assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
- .list(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.contains(CarbonCommonConstants.FACT_FILE_EXT)
- }
- }).length > 0)
- checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
- checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
- sql("drop datamap preagg on table main")
- assertResult(false)(new File(path).exists())
- sql("drop table main")
- }
-
- override def afterAll {
- sql("DROP TABLE IF EXISTS maintable")
- sql("drop table if exists uniqdata")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- sql("drop table if exists datamaptest")
- sql("drop table if exists datamapshowtest")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
new file mode 100644
index 0000000..a05a8c2
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{File, FilenameFilter}
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.common.exceptions.MetadataProcessException
+import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+
+class TestIndexDataMapCommand extends QueryTest with BeforeAndAfterAll {
+
+ val testData = s"$resourcesPath/sample.csv"
+
+ override def beforeAll {
+ sql("drop table if exists datamaptest")
+ sql("drop table if exists datamapshowtest")
+ sql("drop table if exists uniqdata")
+ sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
+ }
+
+ val newClass = "org.apache.spark.sql.CarbonSource"
+
+ test("test datamap create: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
+ }
+ }
+
+ test("test datamap create with dmproperties: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ }
+ }
+
+ test("test datamap create with existing name: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql(
+ s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
+ }
+ }
+
+ test("test datamap create with preagg") {
+ sql("drop datamap if exists datamap3 on table datamaptest")
+ sql(
+ "create datamap datamap3 on table datamaptest using 'preaggregate' as select count(a) from datamaptest")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 1)
+ assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
+ assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
+ }
+
+ test("check hivemetastore after drop datamap") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ "true")
+ sql("drop table if exists hiveMetaStoreTable")
+ sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
+
+ sql(
+ "create datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable using 'preaggregate' as select count(a) from hiveMetaStoreTable")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable"), true, "datamap_hiveMetaStoreTable")
+
+ sql("drop datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable"), false, "datamap_hiveMetaStoreTable")
+
+ } finally {
+ sql("drop table hiveMetaStoreTable")
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("drop the table having pre-aggregate") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ "true")
+ sql("drop table if exists hiveMetaStoreTable_1")
+ sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
+
+ sql(
+ "create datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1 using 'preaggregate' as select count(a) from hiveMetaStoreTable_1")
+
+ checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
+ true,
+ "datamap_hiveMetaStoreTable_1")
+
+ sql("drop datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
+ checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
+ false,
+ "datamap_hiveMetaStoreTable_1")
+ assert(sql("show datamap on table hiveMetaStoreTable_1").collect().length == 0)
+ sql("drop table hiveMetaStoreTable_1")
+
+ checkExistence(sql("show tables"), false, "datamap_hiveMetaStoreTable_1")
+ }
+ finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("test datamap create with preagg with duplicate name") {
+ sql(
+ s"""
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | AS SELECT COUNT(a) FROM datamaptest
+ """.stripMargin)
+ }
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ }
+
+ test("test drop non-exist datamap") {
+ intercept[NoSuchDataMapException] {
+ sql("drop datamap nonexist on table datamaptest")
+ }
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
+ assert(table != null)
+ val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
+ assert(dataMapSchemaList.size() == 2)
+ }
+
+ test("test show datamap without preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' ")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
+ }
+ }
+
+ test("test show datamap with preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 2)
+ checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
+ }
+ }
+
+ test("test show datamap with no datamap") {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ assert(sql("show datamap on table datamapshowtest").collect().length == 0)
+ }
+
+ test("test show datamap after dropping datamap: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
+ sql("drop table if exists datamapshowtest")
+ sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
+ sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
+ sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
+ sql("drop datamap datamap1 on table datamapshowtest")
+ val frame = sql("show datamap on table datamapshowtest")
+ assert(frame.collect().length == 1)
+ checkExistence(frame, true, "datamap2", "(NA)", newClass)
+ }
+ }
+
+ test("test if preaggregate load is successfull for hivemetastore") {
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("DROP TABLE IF EXISTS maintable")
+ sql(
+ """
+ | CREATE TABLE maintable(id int, name string, city string, age int)
+ | STORED BY 'org.apache.carbondata.format'
+ """.stripMargin)
+ sql(
+ s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
+
+ .stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
+ checkAnswer(sql(s"select * from maintable_preagg_sum"),
+ Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+ } finally {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+
+ test("test preaggregate load for decimal column for hivemetastore") {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
+ sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+ sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
+ sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
+ checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
+ sql("drop datamap if exists uniqdata_agg on table uniqdata")
+ }
+
+ test("create pre-agg table with path") {
+ sql("drop table if exists main_preagg")
+ sql("drop table if exists main ")
+ val warehouse = s"$metastoredb/warehouse"
+ val path = warehouse + "/" + System.nanoTime + "_preAggTestPath"
+ sql(
+ s"""
+ | create table main(
+ | year int,
+ | month int,
+ | name string,
+ | salary int)
+ | stored by 'carbondata'
+ | tblproperties('sort_columns'='month,year,name')
+ """.stripMargin)
+ sql("insert into main select 10,11,'amy',12")
+ sql("insert into main select 10,11,'amy',14")
+ sql(
+ s"""
+ | create datamap preagg
+ | on table main
+ | using 'preaggregate'
+ | dmproperties ('path'='$path')
+ | as select name,avg(salary)
+ | from main
+ | group by name
+ """.stripMargin)
+ assertResult(true)(new File(path).exists())
+ assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
+ .list(new FilenameFilter {
+ override def accept(dir: File, name: String): Boolean = {
+ name.contains(CarbonCommonConstants.FACT_FILE_EXT)
+ }
+ }).length > 0)
+ checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
+ checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
+ sql("drop datamap preagg on table main")
+ assertResult(false)(new File(path).exists())
+ sql("drop table main")
+ }
+
+ override def afterAll {
+ sql("DROP TABLE IF EXISTS maintable")
+ sql("drop table if exists uniqdata")
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ sql("drop table if exists datamaptest")
+ sql("drop table if exists datamapshowtest")
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
new file mode 100644
index 0000000..2b3a306
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+
+import static org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.PREAGGREGATE;
+import static org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES;
+
+public class DataMapManager {
+
+ private static DataMapManager INSTANCE;
+
+ private DataMapManager() { }
+
+ public static synchronized DataMapManager get() {
+ if (INSTANCE == null) {
+ INSTANCE = new DataMapManager();
+ }
+ return INSTANCE;
+ }
+
+ /**
+ * Return a DataMapProvider instance for specified dataMapSchema.
+ */
+ public DataMapProvider getDataMapProvider(DataMapSchema dataMapSchema) {
+ DataMapProvider provider;
+ if (dataMapSchema.getClassName().equalsIgnoreCase(PREAGGREGATE.toString())) {
+ provider = new PreAggregateDataMapProvider();
+ } else if (dataMapSchema.getClassName().equalsIgnoreCase(TIMESERIES.toString())) {
+ provider = new TimeseriesDataMapProvider();
+ } else {
+ provider = new IndexDataMapProvider();
+ }
+ return provider;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProperty.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProperty.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProperty.java
new file mode 100644
index 0000000..0cf0d04
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProperty.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+/**
+ * Property that can be specified when creating DataMap
+ */
+@InterfaceAudience.Internal
+public class DataMapProperty {
+
+ /**
+ * Used to specify the store location of the datamap
+ */
+ public static final String PATH = "path";
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
new file mode 100644
index 0000000..a71e0d8
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.processing.exception.DataLoadingException;
+
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * DataMap is a accelerator for certain type of query. Developer can add new DataMap
+ * implementation to improve query performance.
+ *
+ * Currently two types of DataMap are supported
+ * <ol>
+ * <li> MVDataMap: materialized view type of DataMap to accelerate olap style query,
+ * like SPJG query (select, predicate, join, groupby) </li>
+ * <li> IndexDataMap: index type of DataMap to accelerate filter query </li>
+ * </ol>
+ *
+ * <p>
+ * In following command <br>
+ * {@code CREATE DATAMAP dm ON TABLE main USING 'provider'}, <br>
+ * the <b>provider</b> string can be a short name or class name of the DataMap implementation.
+ *
+ * <br>Currently CarbonData supports following provider:
+ * <ol>
+ * <li> preaggregate: one type of MVDataMap that do pre-aggregate of single table </li>
+ * <li> timeseries: one type of MVDataMap that do pre-aggregate based on time dimension
+ * of the table </li>
+ * <li> class name of {@link org.apache.carbondata.core.datamap.dev.IndexDataMapFactory}
+ * implementation: Developer can implement new type of IndexDataMap by extending
+ * {@link org.apache.carbondata.core.datamap.dev.IndexDataMapFactory} </li>
+ * </ol>
+ *
+ * @since 1.4.0
+ */
+@InterfaceAudience.Developer("DataMap")
+@InterfaceStability.Unstable
+public interface DataMapProvider {
+
+ /**
+ * Initialize a datamap's metadata.
+ * This is called when user creates datamap, for example "CREATE DATAMAP dm ON TABLE mainTable"
+ * Implementation should initialize metadata for datamap, like creating table
+ */
+ void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
+ SparkSession sparkSession) throws MalformedDataMapCommandException;
+
+ /**
+ * Initialize a datamap's data.
+ * This is called when user creates datamap, for example "CREATE DATAMAP dm ON TABLE mainTable"
+ * Implementation should initialize data for datamap, like creating data folders
+ */
+ void initData(CarbonTable mainTable, SparkSession sparkSession);
+
+ /**
+ * Opposite operation of {@link #initMeta(CarbonTable, DataMapSchema, String, SparkSession)}.
+ * This is called when user drops datamap, for example "DROP DATAMAP dm ON TABLE mainTable"
+ * Implementation should clean all meta for the datamap
+ */
+ void freeMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, SparkSession sparkSession);
+
+ /**
+ * Opposite operation of {@link #initData(CarbonTable, SparkSession)}.
+ * This is called when user drops datamap, for example "DROP DATAMAP dm ON TABLE mainTable"
+ * Implementation should clean all data for the datamap
+ */
+ void freeData(CarbonTable mainTable, DataMapSchema dataMapSchema, SparkSession sparkSession);
+
+ /**
+ * Rebuild the datamap by loading all existing data from mainTable
+ * This is called when refreshing the datamap when
+ * 1. after datamap creation and if `autoRefreshDataMap` is set to true
+ * 2. user manually trigger refresh datamap command
+ */
+ void rebuild(CarbonTable mainTable, SparkSession sparkSession) throws DataLoadingException;
+
+ /**
+ * Build the datamap incrementally by loading specified segment data
+ * This is called when user manually trigger refresh datamap
+ */
+ void incrementalBuild(CarbonTable mainTable, String[] segmentIds, SparkSession sparkSession)
+ throws DataLoadingException;
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
new file mode 100644
index 0000000..e11e522
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.exceptions.MetadataProcessException;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
+import org.apache.carbondata.core.datamap.DataMapRegistry;
+import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.format.TableInfo;
+
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil;
+
+@InterfaceAudience.Internal
+public class IndexDataMapProvider implements DataMapProvider {
+
+ private TableInfo originalTableInfo;
+
+ @Override
+ public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
+ SparkSession sparkSession) throws MalformedDataMapCommandException {
+ IndexDataMapFactory dataMapFactory = createIndexDataMapFactory(dataMapSchema);
+ DataMapStoreManager.getInstance().registerDataMap(
+ mainTable.getAbsoluteTableIdentifier(), dataMapSchema, dataMapFactory);
+ originalTableInfo = PreAggregateUtil.updateMainTable(mainTable, dataMapSchema, sparkSession);
+ }
+
+ @Override
+ public void initData(CarbonTable mainTable, SparkSession sparkSession) {
+ // Nothing is needed to do by default
+ }
+
+ @Override
+ public void freeMeta(CarbonTable mainTable, DataMapSchema dataMapSchema,
+ SparkSession sparkSession) {
+ PreAggregateUtil.updateSchemaInfo(mainTable, originalTableInfo, sparkSession);
+ }
+
+ @Override
+ public void freeData(CarbonTable mainTable, DataMapSchema dataMapSchema,
+ SparkSession sparkSession) {
+ DataMapStoreManager.getInstance().clearDataMap(
+ mainTable.getAbsoluteTableIdentifier(), dataMapSchema.getDataMapName());
+ }
+
+ @Override
+ public void rebuild(CarbonTable mainTable, SparkSession sparkSession) {
+ // Nothing is needed to do by default
+ }
+
+ @Override
+ public void incrementalBuild(CarbonTable mainTable, String[] segmentIds,
+ SparkSession sparkSession) {
+ throw new UnsupportedOperationException();
+ }
+
+ private IndexDataMapFactory createIndexDataMapFactory(DataMapSchema dataMapSchema)
+ throws MalformedDataMapCommandException {
+ IndexDataMapFactory dataMapFactory;
+ try {
+ // try to create DataMapProvider instance by taking providerName as class name
+ Class<? extends IndexDataMapFactory> providerClass =
+ (Class<? extends IndexDataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ dataMapFactory = providerClass.newInstance();
+ } catch (ClassNotFoundException e) {
+ // try to create DataMapProvider instance by taking providerName as short name
+ dataMapFactory = getDataMapFactoryByShortName(dataMapSchema.getClassName());
+ } catch (Throwable e) {
+ throw new MetadataProcessException(
+ "failed to create DataMapProvider '" + dataMapSchema.getClassName() + "'", e);
+ }
+ return dataMapFactory;
+ }
+
+ private IndexDataMapFactory getDataMapFactoryByShortName(String providerName)
+ throws MalformedDataMapCommandException {
+ IndexDataMapFactory dataMapFactory;
+ String className = DataMapRegistry.getDataMapClassName(providerName);
+ if (className != null) {
+ try {
+ Class<? extends IndexDataMapFactory> datamapClass =
+ (Class<? extends IndexDataMapFactory>) Class.forName(providerName);
+ dataMapFactory = datamapClass.newInstance();
+ } catch (ClassNotFoundException ex) {
+ throw new MalformedDataMapCommandException(
+ "DataMap '" + providerName + "' not found", ex);
+ } catch (Throwable ex) {
+ throw new MetadataProcessException(
+ "failed to create DataMap '" + providerName + "'", ex);
+ }
+ } else {
+ throw new MalformedDataMapCommandException(
+ "DataMap '" + providerName + "' not found");
+ }
+ return dataMapFactory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
new file mode 100644
index 0000000..dc53900
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateTableHelper;
+import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand;
+import scala.Some;
+
+@InterfaceAudience.Internal
+public class PreAggregateDataMapProvider implements DataMapProvider {
+ protected PreAggregateTableHelper helper;
+ protected CarbonDropTableCommand dropTableCommand;
+
+ @Override
+ public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
+ SparkSession sparkSession) throws MalformedDataMapCommandException {
+ validateDmProperty(dataMapSchema);
+ helper = new PreAggregateTableHelper(
+ mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getClassName(),
+ dataMapSchema.getProperties(), ctasSqlStatement, null, false);
+ helper.initMeta(sparkSession);
+ }
+
+ private void validateDmProperty(DataMapSchema dataMapSchema)
+ throws MalformedDataMapCommandException {
+ if (!dataMapSchema.getProperties().isEmpty()) {
+ if (dataMapSchema.getProperties().size() > 1 ||
+ !dataMapSchema.getProperties().containsKey(DataMapProperty.PATH)) {
+ throw new MalformedDataMapCommandException(
+ "Only 'path' dmproperty is allowed for this datamap");
+ }
+ }
+ }
+
+ @Override
+ public void initData(CarbonTable mainTable, SparkSession sparkSession) {
+ // Nothing is needed to do by default
+ }
+
+ @Override
+ public void freeMeta(CarbonTable mainTable, DataMapSchema dataMapSchema,
+ SparkSession sparkSession) {
+ dropTableCommand = new CarbonDropTableCommand(
+ true,
+ new Some<>(dataMapSchema.getRelationIdentifier().getDatabaseName()),
+ dataMapSchema.getRelationIdentifier().getTableName(),
+ true);
+ dropTableCommand.processMetadata(sparkSession);
+ }
+
+ @Override
+ public void freeData(CarbonTable mainTable, DataMapSchema dataMapSchema,
+ SparkSession sparkSession) {
+ if (dropTableCommand != null) {
+ dropTableCommand.processData(sparkSession);
+ }
+ }
+
+ @Override
+ public void rebuild(CarbonTable mainTable, SparkSession sparkSession) {
+ if (helper != null) {
+ helper.initData(sparkSession);
+ }
+ }
+
+ @Override
+ public void incrementalBuild(CarbonTable mainTable, String[] segmentIds,
+ SparkSession sparkSession) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
new file mode 100644
index 0000000..a66f26a
--- /dev/null
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap;
+
+import java.util.Map;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateTableHelper;
+import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil;
+import scala.Some;
+import scala.Tuple2;
+
+@InterfaceAudience.Internal
+public class TimeseriesDataMapProvider extends PreAggregateDataMapProvider {
+
+ @Override
+ public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
+ SparkSession sparkSession) {
+ Map<String, String> dmProperties = dataMapSchema.getProperties();
+ String dmProviderName = dataMapSchema.getClassName();
+ TimeSeriesUtil.validateTimeSeriesGranularity(dmProperties, dmProviderName);
+ Tuple2<String, String> details =
+ TimeSeriesUtil.getTimeSeriesGranularityDetails(dmProperties, dmProviderName);
+ dmProperties.remove(details._1());
+ helper = new PreAggregateTableHelper(
+ mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getClassName(),
+ dmProperties, ctasSqlStatement, new Some(details._1()), false);
+ helper.initMeta(sparkSession);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 39e73ee..9315208 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -94,7 +94,7 @@ class CarbonEnv {
properties.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
}
LOGGER.info(s"carbon env initial: $storePath")
- // trigger event for CarbonEnv init
+ // trigger event for CarbonEnv create
val operationContext = new OperationContext
val carbonEnvInitPreEvent: CarbonEnvInitPreEvent =
CarbonEnvInitPreEvent(sparkSession, carbonSessionInfo, storePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index c6d86b3..8532e9d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -19,20 +19,13 @@ package org.apache.spark.sql.execution.command.datamap
import scala.collection.JavaConverters._
import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.preaaggregate.{CreatePreAggregateTableCommand, PreAggregateUtil}
-import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
-import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
+import org.apache.carbondata.datamap.{DataMapManager, DataMapProvider}
/**
* Below command class will be used to create datamap on table
@@ -47,59 +40,25 @@ case class CarbonCreateDataMapCommand(
ifNotExistsSet: Boolean = false)
extends AtomicRunnableCommand {
- var createPreAggregateTableCommands: CreatePreAggregateTableCommand = _
- var tableIsExists: Boolean = false
+ private var dataMapProvider: DataMapProvider = _
+ private var mainTable: CarbonTable = _
+ private var dataMapSchema: DataMapSchema = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
// since streaming segment does not support building index and pre-aggregate yet,
// so streaming table does not support create datamap
- val carbonTable =
- CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
- if (carbonTable.isStreamingTable) {
+ mainTable =
+ CarbonEnv.getCarbonTable(tableIdentifier.database, tableIdentifier.table)(sparkSession)
+ if (mainTable.isStreamingTable) {
throw new MalformedCarbonCommandException("Streaming table does not support creating datamap")
}
- validateDataMapName(carbonTable)
- if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
- dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
- TimeSeriesUtil.validateTimeSeriesGranularity(dmProperties, dmClassName)
- createPreAggregateTableCommands = if (dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
- val details = TimeSeriesUtil.getTimeSeriesGranularityDetails(dmProperties, dmClassName)
- val updatedDmProperties = dmProperties - details._1
- CreatePreAggregateTableCommand(
- dataMapName,
- tableIdentifier,
- DataMapProvider.TIMESERIES,
- updatedDmProperties,
- queryString.get,
- Some(details._1))
- } else {
- CreatePreAggregateTableCommand(
- dataMapName,
- tableIdentifier,
- DataMapProvider.PREAGGREGATE,
- dmProperties,
- queryString.get
- )
- }
- try {
- createPreAggregateTableCommands.processMetadata(sparkSession)
- } catch {
- case e: Throwable => throw new MetadataProcessException(s"Failed to create datamap " +
- s"'$dataMapName'", e)
- }
- } else {
- val dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
- dataMapSchema.setProperties(new java.util.HashMap[String, String](dmProperties.asJava))
- val dbName = CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
- Some(dbName),
- tableIdentifier.table)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- carbonTable.getAbsoluteTableIdentifier, dataMapSchema)
- // Save DataMapSchema in the schema file of main table
- PreAggregateUtil.updateMainTable(carbonTable, dataMapSchema, sparkSession)
- }
+ validateDataMapName(mainTable)
+ dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
+ dataMapSchema.setProperties(new java.util.HashMap[String, String](dmProperties.asJava))
+ dataMapProvider = DataMapManager.get().getDataMapProvider(dataMapSchema)
+ dataMapProvider.initMeta(mainTable, dataMapSchema, queryString.orNull, sparkSession)
+
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${tableIdentifier.table}")
Seq.empty
@@ -115,29 +74,20 @@ case class CarbonCreateDataMapCommand(
}
override def processData(sparkSession: SparkSession): Seq[Row] = {
- if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
- dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
- if (!tableIsExists) {
- createPreAggregateTableCommands.processData(sparkSession)
- } else {
- Seq.empty
+ if (dataMapProvider != null) {
+ dataMapProvider.initData(mainTable, sparkSession)
+ if (mainTable.isAutoRefreshDataMap) {
+ dataMapProvider.rebuild(mainTable, sparkSession)
}
- } else {
- Seq.empty
}
+ Seq.empty
}
override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
- if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
- dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
- if (!tableIsExists && createPreAggregateTableCommands != null) {
- createPreAggregateTableCommands.undoMetadata(sparkSession, exception)
- } else {
- Seq.empty
- }
- } else {
- throw new MalformedDataMapCommandException("Unknown datamap provider/class " + dmClassName)
+ if (dataMapProvider != null) {
+ dataMapProvider.freeMeta(mainTable, dataMapSchema, sparkSession)
}
+ Seq.empty
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index e89d15a..f773a55 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -25,16 +25,15 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
-import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, NoSuchDataMapException}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
+import org.apache.carbondata.datamap.{DataMapManager, DataMapProvider}
import org.apache.carbondata.events._
/**
@@ -52,8 +51,10 @@ case class CarbonDropDataMapCommand(
forceDrop: Boolean = false)
extends AtomicRunnableCommand {
- val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- var commandToRun: CarbonDropTableCommand = _
+ private val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ private var dataMapProvider: DataMapProvider = _
+ private var mainTable: CarbonTable = _
+ private var dataMapSchema: DataMapSchema = _
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val dbName = CarbonEnv.getDatabaseName(databaseNameOp)(sparkSession)
@@ -66,7 +67,6 @@ case class CarbonDropDataMapCommand(
catalog.checkSchemasModifiedTimeAndReloadTable(TableIdentifier(tableName, Some(dbName)))
val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
try {
- forceDropTableFromMetaStore(sparkSession)
locksToBeAcquired foreach {
lock => carbonLocks += CarbonLockUtil.getLockObject(tableIdentifier, lock)
}
@@ -82,35 +82,33 @@ case class CarbonDropDataMapCommand(
// If force drop is true then remove the datamap from hivemetastore. No need to remove from
// parent as the first condition would have taken care of it.
if (carbonTable.isDefined && carbonTable.get.getTableInfo.getDataMapSchemaList.size() > 0) {
- val dataMapSchema = carbonTable.get.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
+ mainTable = carbonTable.get
+ val dataMapSchemaOp = mainTable.getTableInfo.getDataMapSchemaList.asScala.zipWithIndex.
find(_._1.getDataMapName.equalsIgnoreCase(dataMapName))
- if (dataMapSchema.isDefined) {
+ if (dataMapSchemaOp.isDefined) {
+ dataMapSchema = dataMapSchemaOp.get._1
val operationContext = new OperationContext
val dropDataMapPreEvent =
DropDataMapPreEvent(
- Some(dataMapSchema.get._1),
+ Some(dataMapSchema),
ifExistsSet,
sparkSession)
OperationListenerBus.getInstance.fireEvent(dropDataMapPreEvent, operationContext)
- carbonTable.get.getTableInfo.getDataMapSchemaList.remove(dataMapSchema.get._2)
+ mainTable.getTableInfo.getDataMapSchemaList.remove(dataMapSchemaOp.get._2)
val schemaConverter = new ThriftWrapperSchemaConverterImpl
PreAggregateUtil.updateSchemaInfo(
- carbonTable.get,
+ mainTable,
schemaConverter.fromWrapperToExternalTableInfo(
- carbonTable.get.getTableInfo,
+ mainTable.getTableInfo,
dbName,
tableName))(sparkSession)
- commandToRun = CarbonDropTableCommand(
- ifExistsSet = true,
- Some(dataMapSchema.get._1.getRelationIdentifier.getDatabaseName),
- dataMapSchema.get._1.getRelationIdentifier.getTableName,
- dropChildTable = true
- )
- commandToRun.processMetadata(sparkSession)
+ dataMapProvider = DataMapManager.get.getDataMapProvider(dataMapSchema)
+ dataMapProvider.freeMeta(mainTable, dataMapSchema, sparkSession)
+
// fires the event after dropping datamap from main table schema
val dropDataMapPostEvent =
DropDataMapPostEvent(
- Some(dataMapSchema.get._1),
+ Some(dataMapSchema),
ifExistsSet,
sparkSession)
OperationListenerBus.getInstance.fireEvent(dropDataMapPostEvent, operationContext)
@@ -118,7 +116,7 @@ case class CarbonDropDataMapCommand(
throw new NoSuchDataMapException(dataMapName, tableName)
}
} else if (carbonTable.isDefined &&
- carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0) {
+ carbonTable.get.getTableInfo.getDataMapSchemaList.size() == 0) {
if (!ifExistsSet) {
throw new NoSuchDataMapException(dataMapName, tableName)
}
@@ -142,39 +140,10 @@ case class CarbonDropDataMapCommand(
Seq.empty
}
- /**
- * Used to drop child datamap from hive metastore if it exists.
- * forceDrop will be true only when an exception occurs in main table status updation for
- * parent table or in processData from CreatePreAggregateTableCommand.
- */
- private def forceDropTableFromMetaStore(sparkSession: SparkSession): Unit = {
- if (forceDrop) {
- val childTableName = tableName + "_" + dataMapName
- LOGGER.info(s"Trying to force drop $childTableName from metastore")
- val childCarbonTable: Option[CarbonTable] = try {
- Some(CarbonEnv.getCarbonTable(databaseNameOp, childTableName)(sparkSession))
- } catch {
- case _: Exception =>
- LOGGER.warn(s"Child table $childTableName not found in metastore")
- None
- }
- if (childCarbonTable.isDefined) {
- commandToRun = CarbonDropTableCommand(
- ifExistsSet = true,
- Some(childCarbonTable.get.getDatabaseName),
- childCarbonTable.get.getTableName,
- dropChildTable = true)
- commandToRun.processMetadata(sparkSession)
- }
- }
- }
-
override def processData(sparkSession: SparkSession): Seq[Row] = {
// delete the table folder
- if (commandToRun != null) {
- DataMapStoreManager.getInstance().clearDataMap(
- commandToRun.carbonTable.getAbsoluteTableIdentifier, dataMapName)
- commandToRun.processData(sparkSession)
+ if (dataMapProvider != null) {
+ dataMapProvider.freeData(mainTable, dataMapSchema, sparkSession)
}
Seq.empty
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index c1f86ef..705455f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -93,7 +93,7 @@ case class CarbonAlterTableDropPartitionCommand(
partitionInfo.dropPartition(partitionIndex)
// read TableInfo
- val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)(sparkSession)
+ val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo,
dbName, tableName, tablePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index efd1216..99691d2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -88,7 +88,7 @@ case class CarbonAlterTableSplitPartitionCommand(
updatePartitionInfo(partitionInfo, partitionIds)
// read TableInfo
- val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)(sparkSession)
+ val tableInfo = carbonMetaStore.getThriftTableInfo(carbonTable)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo,
dbName, tableName, tablePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
deleted file mode 100644
index c02ac4f..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.command.preaaggregate
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import org.apache.spark.sql._
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.datamap.CarbonDropDataMapCommand
-import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
-import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
-import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
-import org.apache.spark.sql.optimizer.CarbonFilters
-import org.apache.spark.sql.parser.CarbonSpark2SqlParser
-
-import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
-import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-
-/**
- * Below command class will be used to create pre-aggregate table
- * and updating the parent table about the child table information
- * It will be either success or nothing happen in case of failure:
- * 1. failed to create pre aggregate table.
- * 2. failed to update main table
- *
- */
-case class CreatePreAggregateTableCommand(
- dataMapName: String,
- parentTableIdentifier: TableIdentifier,
- dataMapProvider: DataMapProvider,
- dmProperties: Map[String, String],
- queryString: String,
- timeSeriesFunction: Option[String] = None,
- ifNotExistsSet: Boolean = false)
- extends AtomicRunnableCommand {
-
- var parentTable: CarbonTable = _
- var loadCommand: CarbonLoadDataCommand = _
-
- override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
- val updatedQuery = new CarbonSpark2SqlParser().addPreAggFunction(queryString)
- val df = sparkSession.sql(updatedQuery)
- val fieldRelationMap = PreAggregateUtil.validateActualSelectPlanAndGetAttributes(
- df.logicalPlan, queryString)
- val fields = fieldRelationMap.keySet.toSeq
- val tableProperties = mutable.Map[String, String]()
- dmProperties.foreach(t => tableProperties.put(t._1, t._2))
-
- parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
- if (!parentTable.getTableName.equalsIgnoreCase(parentTableIdentifier.table)) {
- throw new MalformedDataMapCommandException(
- "Parent table name is different in select and create")
- }
- var neworder = Seq[String]()
- val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala
- parentOrder.foreach(parentcol =>
- fields.filter(col => (fieldRelationMap.get(col).get.aggregateFunction.isEmpty) &&
- (parentcol.equals(fieldRelationMap.get(col).get.
- columnTableRelationList.get(0).parentColumnName)))
- .map(cols => neworder :+= cols.column)
- )
- tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, neworder.mkString(","))
- tableProperties.put("sort_scope", parentTable.getTableInfo.getFactTable.
- getTableProperties.asScala.getOrElse("sort_scope", CarbonCommonConstants
- .LOAD_SORT_SCOPE_DEFAULT))
- tableProperties
- .put(CarbonCommonConstants.TABLE_BLOCKSIZE, parentTable.getBlockSizeInMB.toString)
- val tableIdentifier =
- TableIdentifier(parentTableIdentifier.table + "_" + dataMapName,
- parentTableIdentifier.database)
- // prepare table model of the collected tokens
- val tableModel: TableModel = new CarbonSpark2SqlParser().prepareTableModel(
- ifNotExistPresent = ifNotExistsSet,
- new CarbonSpark2SqlParser().convertDbNameToLowerCase(tableIdentifier.database),
- tableIdentifier.table.toLowerCase,
- fields,
- Seq(),
- tableProperties,
- None,
- isAlterFlow = false,
- None)
- // updating the relation identifier, this will be stored in child table
- // which can be used during dropping of pre-aggreate table as parent table will
- // also get updated
- if(timeSeriesFunction.isDefined) {
- TimeSeriesUtil.validateTimeSeriesEventTime(dmProperties, parentTable)
- TimeSeriesUtil.validateEventTimeColumnExitsInSelect(
- fieldRelationMap,
- dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get)
- TimeSeriesUtil.updateTimeColumnSelect(fieldRelationMap,
- dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get,
- timeSeriesFunction.get)
- }
- tableModel.parentTable = Some(parentTable)
- tableModel.dataMapRelation = Some(fieldRelationMap)
- val tablePath = if (dmProperties.contains("path")) {
- dmProperties("path")
- } else {
- CarbonEnv.getTablePath(tableModel.databaseNameOp, tableModel.tableName)(sparkSession)
- }
- CarbonCreateTableCommand(TableNewProcessor(tableModel),
- tableModel.ifNotExistsSet, Some(tablePath), isVisible = false).run(sparkSession)
-
- val table = CarbonEnv.getCarbonTable(tableIdentifier)(sparkSession)
- val tableInfo = table.getTableInfo
- // child schema object which will be updated on parent table about the
- val childSchema = tableInfo.getFactTable.buildChildSchema(
- dataMapName,
- dataMapProvider.getClassName,
- tableInfo.getDatabaseName,
- queryString,
- "AGGREGATION")
- dmProperties.foreach(f => childSchema.getProperties.put(f._1, f._2))
-
- // updating the parent table about child table
- PreAggregateUtil.updateMainTable(
- parentTable,
- childSchema,
- sparkSession)
- // After updating the parent carbon table with data map entry extract the latest table object
- // to be used in further create process.
- parentTable = CarbonEnv.getCarbonTable(parentTableIdentifier.database,
- parentTableIdentifier.table)(sparkSession)
- val updatedLoadQuery = if (timeSeriesFunction.isDefined) {
- PreAggregateUtil.createTimeSeriesSelectQueryFromMain(childSchema.getChildSchema,
- parentTable.getTableName,
- parentTable.getDatabaseName)
- } else {
- queryString
- }
- val dataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
- updatedLoadQuery)).drop("preAggLoad")
- loadCommand = PreAggregateUtil.createLoadCommandForChild(
- childSchema.getChildSchema.getListOfColumns,
- tableIdentifier,
- dataFrame,
- false,
- sparkSession = sparkSession)
- loadCommand.processMetadata(sparkSession)
- Seq.empty
- }
-
- override def undoMetadata(sparkSession: SparkSession, exception: Exception): Seq[Row] = {
- // drop child table and undo the change in table info of main table
- CarbonDropDataMapCommand(
- dataMapName,
- ifExistsSet = true,
- parentTableIdentifier.database,
- parentTableIdentifier.table,
- forceDrop = true).run(sparkSession)
- Seq.empty
- }
-
- override def processData(sparkSession: SparkSession): Seq[Row] = {
- // load child table if parent table has existing segments
- // This will be used to check if the parent table has any segments or not. If not then no
- // need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
- // table.
- SegmentStatusManager.deleteLoadsAndUpdateMetadata(parentTable, false)
- val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
- if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
- load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
- throw new UnsupportedOperationException(
- "Cannot create pre-aggregate table when insert is in progress on main table")
- } else if (loadAvailable.nonEmpty) {
- // Passing segmentToLoad as * because we want to load all the segments into the
- // pre-aggregate table even if the user has set some segments on the parent table.
- loadCommand.dataFrame = Some(PreAggregateUtil
- .getDataFrame(sparkSession, loadCommand.logicalPlan.get))
- PreAggregateUtil.startDataLoadForDataMap(
- TableIdentifier(parentTable.getTableName, Some(parentTable.getDatabaseName)),
- segmentToLoad = "*",
- validateSegments = true,
- loadCommand,
- isOverwrite = false,
- sparkSession)
- }
- Seq.empty
- }
-}
-
-
[25/54] [abbrv] carbondata git commit: [CARBONDATA-2156] Add
interface annotation
Posted by ra...@apache.org.
[CARBONDATA-2156] Add interface annotation
InterfaceAudience and InterfaceStability annotation should be added for user and developer
1.InetfaceAudience can be User and Developer
2.InterfaceStability can be Stable, Evolving, Unstable
This closes #1968
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ce6e71c5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ce6e71c5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ce6e71c5
Branch: refs/heads/master
Commit: ce6e71c5b19876415f00bd20b1cdf7fe5cc22825
Parents: 8d8b589
Author: Jacky Li <ja...@qq.com>
Authored: Sun Feb 11 10:12:10 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../common/annotations/InterfaceAudience.java | 58 ++++++++++++++++
.../common/annotations/InterfaceStability.java | 69 ++++++++++++++++++++
2 files changed, 127 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce6e71c5/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
new file mode 100644
index 0000000..fa9729d
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/**
+ * This annotation is ported and modified from Apache Hadoop project.
+ *
+ * Annotation to inform users of a package, class or method's intended audience.
+ * Currently the audience can be {@link User}, {@link Developer}
+ *
+ * Public classes that are not marked with this annotation must be
+ * considered by default as {@link Developer}.</li>
+ *
+ * External applications must only use classes that are marked {@link User}.
+ *
+ * Methods may have a different annotation that it is more restrictive
+ * compared to the audience classification of the class. Example: A class
+ * might be {@link User}, but a method may be {@link Developer}
+ */
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class InterfaceAudience {
+ /**
+ * Intended for use by any project or application.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface User { }
+
+ /**
+ * Intended only for developers to extend interface for CarbonData project
+ * For example, new Datamap implementations.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Developer { }
+
+ private InterfaceAudience() { } // Audience can't exist on its own
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce6e71c5/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
new file mode 100644
index 0000000..b8e5e52
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience.*;
+
+/**
+ * This annotation is ported and modified from Apache Hadoop project.
+ *
+ * Annotation to inform users of how much to rely on a particular package,
+ * class or method not changing over time. Currently the stability can be
+ * {@link Stable}, {@link Evolving} or {@link Unstable}. <br>
+ *
+ * 1. {@link Stable} means compatibility can break only at major release (m.0)
+ * 2. {@link Evolving} means compatibility can break at minor release (m.x)
+ * 3. {@link Unstable} means compatibility can break at any release
+ *
+ * <ul><li>All classes that are annotated with {@link User} or
+ * {@link Developer} must have InterfaceStability annotation. </li>
+ * <li>Classes that are {@link Private} are to be considered unstable unless
+ * a different InterfaceStability annotation states otherwise.</li>
+ * <li>Incompatible changes must not be made to classes marked as stable.</li>
+ * </ul>
+ */
+@InterfaceAudience.User
+@org.apache.hadoop.classification.InterfaceStability.Evolving
+public class InterfaceStability {
+ /**
+ * Can evolve while retaining compatibility for minor release boundaries.;
+ * can break compatibility only at major release (ie. at m.0).
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Stable { }
+
+ /**
+ * Evolving, but can break compatibility at minor release (i.e. m.x)
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Evolving { }
+
+ /**
+ * No guarantee is provided as to reliability or stability across any
+ * level of release granularity.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Unstable { }
+}
[28/54] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove
carbon-spark dependency in store-sdk module
Posted by ra...@apache.org.
[CARBONDATA-2159] Remove carbon-spark dependency in store-sdk module
To make assembling JAR of store-sdk module, it should not depend on carbon-spark module
This closes #1970
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89cfd8e0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89cfd8e0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89cfd8e0
Branch: refs/heads/master
Commit: 89cfd8e06e783c610ff8cf28e86c8501b450da37
Parents: 1d827c7
Author: Jacky Li <ja...@qq.com>
Authored: Sun Feb 11 21:37:04 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/carbondata/common/Maps.java | 39 ++
.../org/apache/carbondata/common/Strings.java | 3 +
.../exceptions/TableStatusLockException.java | 34 ++
.../sql/InvalidLoadOptionException.java | 33 +
.../sql/MalformedCarbonCommandException.java | 75 +++
.../sql/MalformedDataMapCommandException.java | 37 ++
.../exceptions/sql/NoSuchDataMapException.java | 39 ++
.../carbondata/core/datamap/TableDataMap.java | 5 +-
.../exception/ConcurrentOperationException.java | 50 ++
.../statusmanager/SegmentStatusManager.java | 124 ++++
.../carbondata/core/util/DeleteLoadFolders.java | 210 +++++++
.../preaggregate/TestPreAggCreateCommand.scala | 2 +-
.../preaggregate/TestPreAggregateDrop.scala | 2 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 2 +-
.../timeseries/TestTimeSeriesDropSuite.scala | 2 +-
.../TestTimeseriesTableSelection.scala | 2 +-
.../TestDataLoadWithColumnsMoreThanSchema.scala | 3 +-
.../dataload/TestGlobalSortDataLoad.scala | 2 +-
.../TestLoadDataWithDiffTimestampFormat.scala | 2 +-
.../TestLoadDataWithFileHeaderException.scala | 11 +-
...ataWithMalformedCarbonCommandException.scala | 3 +-
.../testsuite/dataload/TestLoadOptions.scala | 2 +-
.../dataload/TestTableLevelBlockSize.scala | 4 +-
.../testsuite/datamap/TestDataMapCommand.scala | 2 +-
.../dataretention/DataRetentionTestCase.scala | 2 +-
.../spark/testsuite/datetype/DateTypeTest.scala | 2 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 3 +-
integration/spark-common/pom.xml | 5 -
.../exception/ConcurrentOperationException.java | 38 --
.../MalformedCarbonCommandException.java | 69 ---
.../MalformedDataMapCommandException.java | 32 -
.../spark/exception/NoSuchDataMapException.java | 33 -
.../org/apache/carbondata/api/CarbonStore.scala | 3 +-
.../spark/CarbonColumnValidator.scala | 8 +-
.../carbondata/spark/load/ValidateUtil.scala | 72 ---
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 6 +-
.../carbondata/spark/util/CommonUtil.scala | 70 +--
.../carbondata/spark/util/DataLoadingUtil.scala | 610 -------------------
.../spark/util/GlobalDictionaryUtil.scala | 2 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +-
.../spark/rdd/CarbonTableCompactor.scala | 2 +-
.../org/apache/spark/sql/CarbonSource.scala | 2 +-
.../datamap/CarbonCreateDataMapCommand.scala | 2 +-
.../datamap/CarbonDropDataMapCommand.scala | 2 +-
.../CarbonAlterTableCompactionCommand.scala | 13 +-
.../management/CarbonLoadDataCommand.scala | 17 +-
.../CarbonProjectForDeleteCommand.scala | 2 +-
.../CarbonProjectForUpdateCommand.scala | 2 +-
.../command/mutation/IUDCommonUtil.scala | 2 +-
.../CreatePreAggregateTableCommand.scala | 7 +-
.../preaaggregate/PreAggregateUtil.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 3 +-
.../command/timeseries/TimeSeriesUtil.scala | 2 +-
.../datasources/CarbonFileFormat.scala | 14 +-
.../sql/execution/strategy/DDLStrategy.scala | 2 +-
.../strategy/StreamingTableStrategy.scala | 2 +-
.../execution/command/CarbonHiveCommands.scala | 2 +-
.../sql/parser/CarbonSpark2SqlParser.scala | 2 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 2 +-
.../org/apache/spark/util/AlterTableUtil.scala | 2 +-
.../org/apache/spark/util/TableAPIUtil.scala | 2 +-
.../spark/sql/hive/CarbonSessionState.scala | 7 +-
.../segmentreading/TestSegmentReading.scala | 2 +-
.../spark/util/AllDictionaryTestCase.scala | 4 +-
.../util/ExternalColumnDictionaryTestCase.scala | 6 +-
.../TestStreamingTableOperation.scala | 4 +-
.../bucketing/TableBucketingTestCase.scala | 2 +-
.../vectorreader/AddColumnTestCases.scala | 2 +-
.../loading/model/CarbonLoadModel.java | 14 +-
.../loading/model/CarbonLoadModelBuilder.java | 322 ++++++++++
.../processing/loading/model/LoadOption.java | 251 ++++++++
.../processing/util/CarbonLoaderUtil.java | 41 +-
.../processing/util/DeleteLoadFolders.java | 210 -------
store/sdk/pom.xml | 2 +-
.../sdk/file/CarbonWriterBuilder.java | 15 +-
.../sdk/file/CSVCarbonWriterSuite.java | 2 +-
.../streaming/StreamSinkFactory.scala | 11 +-
78 files changed, 1344 insertions(+), 1286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/Maps.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/Maps.java b/common/src/main/java/org/apache/carbondata/common/Maps.java
new file mode 100644
index 0000000..14fc329
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/Maps.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common;
+
+import java.util.Map;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+@InterfaceAudience.Developer
+public class Maps {
+
+ /**
+ * Return value if key is contained in the map, else return defauleValue.
+ * This is added to avoid JDK 8 dependency
+ */
+ public static <K, V> V getOrDefault(Map<K, V> map, K key, V defaultValue) {
+ V value = map.get(key);
+ if (value != null) {
+ return value;
+ } else {
+ return defaultValue;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/Strings.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/Strings.java b/common/src/main/java/org/apache/carbondata/common/Strings.java
index 23288dd..08fdc3c 100644
--- a/common/src/main/java/org/apache/carbondata/common/Strings.java
+++ b/common/src/main/java/org/apache/carbondata/common/Strings.java
@@ -19,6 +19,9 @@ package org.apache.carbondata.common;
import java.util.Objects;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+@InterfaceAudience.Developer
public class Strings {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/exceptions/TableStatusLockException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/TableStatusLockException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/TableStatusLockException.java
new file mode 100644
index 0000000..89cfd46
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/TableStatusLockException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown when failed to acquire lock for table status metadata,
+ * or re-try timed out
+ */
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class TableStatusLockException extends RuntimeException {
+
+ public TableStatusLockException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/exceptions/sql/InvalidLoadOptionException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/InvalidLoadOptionException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/InvalidLoadOptionException.java
new file mode 100644
index 0000000..41b2434
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/InvalidLoadOptionException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions.sql;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown when loading option is invalid for SQL
+ * loading statement (LOAD DATA, INSERT INTO)
+ */
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class InvalidLoadOptionException extends MalformedCarbonCommandException {
+ public InvalidLoadOptionException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
new file mode 100644
index 0000000..5fe3ce8
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedCarbonCommandException.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions.sql;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown if any validation fails then parsing
+ * SQL statement.
+ */
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class MalformedCarbonCommandException extends Exception {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public MalformedCarbonCommandException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public MalformedCarbonCommandException(String msg, Throwable t) {
+ super(msg, t);
+ this.msg = msg;
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override
+ public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
new file mode 100644
index 0000000..7c25b2c
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions.sql;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown when Datamap related SQL statement is invalid
+ */
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class MalformedDataMapCommandException extends MalformedCarbonCommandException {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public MalformedDataMapCommandException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/common/src/main/java/org/apache/carbondata/common/exceptions/sql/NoSuchDataMapException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/NoSuchDataMapException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/NoSuchDataMapException.java
new file mode 100644
index 0000000..7ab9048
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/NoSuchDataMapException.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions.sql;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * This exception will be thrown if datamap is not found when executing datamap
+ * related SQL statement
+ */
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class NoSuchDataMapException extends MalformedCarbonCommandException {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ public NoSuchDataMapException(String dataMapName, String tableName) {
+ super("Datamap with name " + dataMapName + " does not exist under table " + tableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 67c1f4f..62a9119 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -29,8 +29,8 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -160,7 +160,8 @@ public final class TableDataMap extends OperationEventListener {
writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
detailedBlocklet.setDataMapWriterPath(blockletwritePath);
serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
- }detailedBlocklet.setSegmentId(distributable.getSegment().getSegmentNo());
+ }
+ detailedBlocklet.setSegmentId(distributable.getSegment().getSegmentNo());
detailedBlocklets.add(detailedBlocklet);
}
return detailedBlocklets;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java b/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
new file mode 100644
index 0000000..7e717ba
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/exception/ConcurrentOperationException.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.exception;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+/**
+ * This exception will be thrown when executing concurrent operations which
+ * is not supported in carbon.
+ *
+ * For example, when INSERT OVERWRITE is executing, other operations are not
+ * allowed, so this exception will be thrown
+ */
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class ConcurrentOperationException extends Exception {
+
+ public ConcurrentOperationException(String dbName, String tableName, String command1,
+ String command2) {
+ super(command1 + " is in progress for table " + dbName + "." + tableName + ", " + command2 +
+ " operation is not allowed");
+ }
+
+ public ConcurrentOperationException(CarbonTable table, String command1, String command2) {
+ this(table.getDatabaseName(), table.getTableName(), command1, command2);
+ }
+
+ public String getMessage() {
+ return super.getMessage();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index d3e04ca..b3b1240 100755
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.carbondata.common.exceptions.TableStatusLockException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -47,6 +48,7 @@ import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DeleteLoadFolders;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import com.google.gson.Gson;
@@ -764,4 +766,126 @@ public class SegmentStatusManager {
}
}
+ private static boolean isLoadDeletionRequired(String metaDataLocation) {
+ LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metaDataLocation);
+ if (details != null && details.length > 0) {
+ for (LoadMetadataDetails oneRow : details) {
+ if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus()
+ || SegmentStatus.COMPACTED == oneRow.getSegmentStatus()
+ || SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus()
+ || SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus())
+ && oneRow.getVisibility().equalsIgnoreCase("true")) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This will update the old table status details before clean files to the latest table status.
+ * @param oldList
+ * @param newList
+ * @return
+ */
+ public static List<LoadMetadataDetails> updateLoadMetadataFromOldToNew(
+ LoadMetadataDetails[] oldList, LoadMetadataDetails[] newList) {
+
+ List<LoadMetadataDetails> newListMetadata =
+ new ArrayList<LoadMetadataDetails>(Arrays.asList(newList));
+ for (LoadMetadataDetails oldSegment : oldList) {
+ if ("false".equalsIgnoreCase(oldSegment.getVisibility())) {
+ newListMetadata.get(newListMetadata.indexOf(oldSegment)).setVisibility("false");
+ }
+ }
+ return newListMetadata;
+ }
+
+ private static void writeLoadMetadata(AbsoluteTableIdentifier identifier,
+ List<LoadMetadataDetails> listOfLoadFolderDetails) throws IOException {
+ String dataLoadLocation = CarbonTablePath.getTableStatusFilePath(identifier.getTablePath());
+
+ DataOutputStream dataOutputStream;
+ Gson gsonObjectToWrite = new Gson();
+ BufferedWriter brWriter = null;
+
+ AtomicFileOperations writeOperation =
+ new AtomicFileOperationsImpl(dataLoadLocation, FileFactory.getFileType(dataLoadLocation));
+
+ try {
+
+ dataOutputStream = writeOperation.openForWrite(FileWriteOperation.OVERWRITE);
+ brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+
+ String metadataInstance = gsonObjectToWrite.toJson(listOfLoadFolderDetails.toArray());
+ brWriter.write(metadataInstance);
+ } finally {
+ try {
+ if (null != brWriter) {
+ brWriter.flush();
+ }
+ } catch (Exception e) {
+ LOG.error("error in flushing ");
+
+ }
+ CarbonUtil.closeStreams(brWriter);
+ writeOperation.close();
+ }
+ }
+
+ public static void deleteLoadsAndUpdateMetadata(
+ CarbonTable carbonTable,
+ boolean isForceDeletion) throws IOException {
+ if (isLoadDeletionRequired(carbonTable.getMetadataPath())) {
+ LoadMetadataDetails[] details =
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
+ AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
+ ICarbonLock carbonTableStatusLock = CarbonLockFactory.getCarbonLockObj(
+ identifier, LockUsage.TABLE_STATUS_LOCK);
+
+ // Delete marked loads
+ boolean isUpdationRequired = DeleteLoadFolders.deleteLoadFoldersFromFileSystem(
+ identifier, isForceDeletion, details, carbonTable.getMetadataPath());
+
+ boolean updationCompletionStatus = false;
+
+ if (isUpdationRequired) {
+ try {
+ // Update load metadate file after cleaning deleted nodes
+ if (carbonTableStatusLock.lockWithRetries()) {
+ LOG.info("Table status lock has been successfully acquired.");
+
+ // read latest table status again.
+ LoadMetadataDetails[] latestMetadata =
+ SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath());
+
+ // update the metadata details from old to new status.
+ List<LoadMetadataDetails> latestStatus =
+ updateLoadMetadataFromOldToNew(details, latestMetadata);
+
+ writeLoadMetadata(identifier, latestStatus);
+ } else {
+ String dbName = identifier.getCarbonTableIdentifier().getDatabaseName();
+ String tableName = identifier.getCarbonTableIdentifier().getTableName();
+ String errorMsg = "Clean files request is failed for " +
+ dbName + "." + tableName +
+ ". Not able to acquire the table status lock due to other operation " +
+ "running in the background.";
+ LOG.audit(errorMsg);
+ LOG.error(errorMsg);
+ throw new TableStatusLockException(errorMsg + " Please try after some time.");
+ }
+ updationCompletionStatus = true;
+ } finally {
+ CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK);
+ if (updationCompletionStatus) {
+ DeleteLoadFolders.physicalFactAndMeasureMetadataDeletion(
+ identifier, carbonTable.getMetadataPath(), isForceDeletion);
+ }
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
new file mode 100644
index 0000000..d1cd4fb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+public final class DeleteLoadFolders {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DeleteLoadFolders.class.getName());
+
+ private DeleteLoadFolders() {
+
+ }
+
+ /**
+ * returns segment path
+ *
+ * @param identifier
+ * @param oneLoad
+ * @return
+ */
+ private static String getSegmentPath(AbsoluteTableIdentifier identifier,
+ LoadMetadataDetails oneLoad) {
+ String segmentId = oneLoad.getLoadName();
+ return CarbonTablePath.getSegmentPath(identifier.getTablePath(), segmentId);
+ }
+
+ public static void physicalFactAndMeasureMetadataDeletion(
+ AbsoluteTableIdentifier absoluteTableIdentifier, String metadataPath, boolean isForceDelete,
+ List<PartitionSpec> specs) {
+ LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
+ for (LoadMetadataDetails oneLoad : currentDetails) {
+ if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
+ try {
+ if (oneLoad.getSegmentFile() != null) {
+ SegmentFileStore
+ .deleteSegment(absoluteTableIdentifier.getTablePath(), oneLoad.getSegmentFile(),
+ specs);
+ } else {
+ String path = getSegmentPath(absoluteTableIdentifier, oneLoad);
+ boolean status = false;
+ if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) {
+ CarbonFile file = FileFactory.getCarbonFile(path, FileFactory.getFileType(path));
+ CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
+
+ @Override public boolean accept(CarbonFile file) {
+ return (CarbonTablePath.isCarbonDataFile(file.getName()) ||
+ CarbonTablePath.isCarbonIndexFile(file.getName()));
+ }
+ });
+
+ //if there are no fact and msr metadata files present then no need to keep
+ //entry in metadata.
+ if (filesToBeDeleted.length == 0) {
+ status = true;
+ } else {
+
+ for (CarbonFile eachFile : filesToBeDeleted) {
+ if (!eachFile.delete()) {
+ LOGGER.warn("Unable to delete the file as per delete command " + eachFile
+ .getAbsolutePath());
+ status = false;
+ } else {
+ status = true;
+ }
+ }
+ }
+ // need to delete the complete folder.
+ if (status) {
+ if (!file.delete()) {
+ LOGGER.warn("Unable to delete the folder as per delete command " + file
+ .getAbsolutePath());
+ }
+ }
+
+ } else {
+ LOGGER.warn("Files are not found in segment " + path
+ + " it seems, files are already being deleted");
+ }
+
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
+ }
+ }
+ }
+ }
+
+ private static boolean checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad,
+ boolean isForceDelete) {
+ if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() ||
+ SegmentStatus.COMPACTED == oneLoad.getSegmentStatus() ||
+ SegmentStatus.INSERT_IN_PROGRESS == oneLoad.getSegmentStatus() ||
+ SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneLoad.getSegmentStatus())
+ && oneLoad.getVisibility().equalsIgnoreCase("true")) {
+ if (isForceDelete) {
+ return true;
+ }
+ long deletionTime = oneLoad.getModificationOrdeletionTimesStamp();
+
+ return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime);
+
+ }
+
+ return false;
+ }
+
+ private static boolean checkIfLoadCanBeDeletedPhysically(LoadMetadataDetails oneLoad,
+ boolean isForceDelete) {
+ if ((SegmentStatus.MARKED_FOR_DELETE == oneLoad.getSegmentStatus() ||
+ SegmentStatus.COMPACTED == oneLoad.getSegmentStatus())) {
+ if (isForceDelete) {
+ return true;
+ }
+ long deletionTime = oneLoad.getModificationOrdeletionTimesStamp();
+
+ return CarbonUpdateUtil.isMaxQueryTimeoutExceeded(deletionTime);
+
+ }
+
+ return false;
+ }
+
+ private static LoadMetadataDetails getCurrentLoadStatusOfSegment(String segmentId,
+ String metadataPath) {
+ LoadMetadataDetails[] currentDetails = SegmentStatusManager.readLoadMetadata(metadataPath);
+ for (LoadMetadataDetails oneLoad : currentDetails) {
+ if (oneLoad.getLoadName().equalsIgnoreCase(segmentId)) {
+ return oneLoad;
+ }
+ }
+ return null;
+ }
+
+ public static boolean deleteLoadFoldersFromFileSystem(
+ AbsoluteTableIdentifier absoluteTableIdentifier, boolean isForceDelete,
+ LoadMetadataDetails[] details, String metadataPath) {
+ boolean isDeleted = false;
+ if (details != null && details.length != 0) {
+ for (LoadMetadataDetails oneLoad : details) {
+ if (checkIfLoadCanBeDeleted(oneLoad, isForceDelete)) {
+ ICarbonLock segmentLock = CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier,
+ CarbonTablePath.addSegmentPrefix(oneLoad.getLoadName()) + LockUsage.LOCK);
+ try {
+ if (oneLoad.getSegmentStatus() == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS
+ || oneLoad.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS) {
+ if (segmentLock.lockWithRetries(1, 5)) {
+ LOGGER.info("Info: Acquired segment lock on segment:" + oneLoad.getLoadName());
+ LoadMetadataDetails currentDetails =
+ getCurrentLoadStatusOfSegment(oneLoad.getLoadName(), metadataPath);
+ if (currentDetails != null && checkIfLoadCanBeDeleted(currentDetails,
+ isForceDelete)) {
+ oneLoad.setVisibility("false");
+ isDeleted = true;
+ LOGGER.info("Info: Deleted the load " + oneLoad.getLoadName());
+ }
+ } else {
+ LOGGER.info("Info: Load in progress for segment" + oneLoad.getLoadName());
+ return isDeleted;
+ }
+ } else {
+ oneLoad.setVisibility("false");
+ isDeleted = true;
+ LOGGER.info("Info: Deleted the load " + oneLoad.getLoadName());
+ }
+ } finally {
+ segmentLock.unlock();
+ LOGGER.info("Info: Segment lock on segment:" + oneLoad.getLoadName() + " is released");
+ }
+ }
+ }
+ }
+ return isDeleted;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index 5b7c310..ccb9b68 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -26,12 +26,12 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
index 911a725..2c7c593 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
@@ -20,7 +20,7 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.spark.exception.NoSuchDataMapException
+import org.apache.carbondata.common.exceptions.sql.NoSuchDataMapException
class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index 3d991a9..43316b3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -20,8 +20,8 @@ import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
-import org.apache.carbondata.spark.exception.{MalformedDataMapCommandException, MalformedCarbonCommandException}
class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
index 545c4de..5fe21e8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala
@@ -19,7 +19,7 @@ package org.apache.carbondata.integration.spark.testsuite.timeseries
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
index 3065952..3f140df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala
@@ -24,8 +24,8 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
index 1532328..4e5ebbb 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala
@@ -18,9 +18,10 @@
package org.apache.carbondata.spark.testsuite.dataload
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
/**
* This class will test data load in which number of columns in data are more than
* the number of columns in schema
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index cbbb191..3babf4f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -23,13 +23,13 @@ import org.apache.commons.io.FileUtils
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.Row
import org.apache.spark.sql.execution.BatchedDataSourceScanExec
import org.apache.spark.sql.test.TestQueryExecutor.projectPath
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.spark.rdd.CarbonScanRDD
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
index ec6fff1..c06d782 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithDiffTimestampFormat.scala
@@ -25,10 +25,10 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
class TestLoadDataWithDiffTimestampFormat extends QueryTest with BeforeAndAfterAll {
val bad_records_action = CarbonProperties.getInstance()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
index 7700ed5..edcdd51 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
@@ -20,6 +20,9 @@ package org.apache.carbondata.spark.testsuite.dataload
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException
+import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
+
class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterAll{
override def beforeAll {
sql("DROP TABLE IF EXISTS t3")
@@ -32,7 +35,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data both file and ddl without file header exception") {
- val e = intercept[Exception] {
+ val e = intercept[CarbonDataLoadingException] {
sql(
s"""LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3""")
}
@@ -41,7 +44,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data ddl provided wrong file header exception") {
- val e = intercept[Exception] {
+ val e = intercept[CarbonDataLoadingException] {
sql(
s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
@@ -52,7 +55,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data with wrong header , but without fileheader") {
- val e = intercept[Exception] {
+ val e = intercept[InvalidLoadOptionException] {
sql(
s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
@@ -63,7 +66,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
test("test load data with wrong header and fileheader") {
- val e = intercept[Exception] {
+ val e = intercept[InvalidLoadOptionException] {
sql(
s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
index 1851705..6759049 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala
@@ -18,9 +18,10 @@
package org.apache.carbondata.spark.testsuite.dataload
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
class TestLoadDataWithMalformedCarbonCommandException extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
index d2c7e63..4ec9335 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
class TestLoadOptions extends QueryTest with BeforeAndAfterAll{
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
index a77b210..f6a049a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestTableLevelBlockSize.scala
@@ -19,11 +19,13 @@ package org.apache.carbondata.spark.testsuite.dataload
import org.apache.spark.sql.Row
import org.scalatest.BeforeAndAfterAll
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
/**
* Test Class for table block size
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index d60b7db..6ac9c7a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -23,11 +23,11 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.spark.exception.MalformedDataMapCommandException
class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index 7c82f75..a70584b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -29,9 +29,9 @@ import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
index e2df07c..b9b01f8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
@@ -16,10 +16,10 @@
*/
package org.apache.carbondata.spark.testsuite.datetype
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
class DateTypeTest extends QueryTest with BeforeAndAfterAll{
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 51df525..17beace 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -21,10 +21,11 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
class TestSortColumns extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/pom.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml
index 005117f..295d62b 100644
--- a/integration/spark-common/pom.xml
+++ b/integration/spark-common/pom.xml
@@ -36,11 +36,6 @@
<dependencies>
<dependency>
<groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-processing</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-hadoop</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ConcurrentOperationException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ConcurrentOperationException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ConcurrentOperationException.java
deleted file mode 100644
index cc0047f..0000000
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/ConcurrentOperationException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.exception;
-
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-
-public class ConcurrentOperationException extends MalformedCarbonCommandException {
-
- public ConcurrentOperationException(String dbName, String tableName, String command1,
- String command2) {
- super(command1 + " is in progress for table " + dbName + "." + tableName + ", " + command2 +
- " operation is not allowed");
- }
-
- public ConcurrentOperationException(CarbonTable table, String command1, String command2) {
- this(table.getDatabaseName(), table.getTableName(), command1, command2);
- }
-
- public String getMessage() {
- return super.getMessage();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedCarbonCommandException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedCarbonCommandException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedCarbonCommandException.java
deleted file mode 100644
index 9f441d3..0000000
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedCarbonCommandException.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.exception;
-
-// After parsing carbon query successfully , if any validation fails then
-// use MalformedCarbonCommandException
-public class MalformedCarbonCommandException extends Exception {
-
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public MalformedCarbonCommandException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public MalformedCarbonCommandException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * getLocalizedMessage
- */
- @Override
- public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
deleted file mode 100644
index a05d8e6..0000000
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/MalformedDataMapCommandException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.exception;
-
-/**
- * Throw exception when using illegal argument
- */
-public class MalformedDataMapCommandException extends MalformedCarbonCommandException {
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- public MalformedDataMapCommandException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
deleted file mode 100644
index 959e70d..0000000
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/exception/NoSuchDataMapException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.exception;
-
-/**
- * if the dataMap does not exist, carbon should throw NoSuchDataMapException
- */
-public class NoSuchDataMapException extends MalformedCarbonCommandException {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- public NoSuchDataMapException(String dataMapName, String tableName) {
- super("Datamap with name " + dataMapName + " does not exist under table " + tableName);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 8adcb11..b69ec37 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.unsafe.types.UTF8String
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -35,7 +36,6 @@ import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, SegmentFile
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.DataLoadingUtil
object CarbonStore {
@@ -139,6 +139,7 @@ object CarbonStore {
carbonCleanFilesLock =
CarbonLockUtil
.getLockObject(absoluteTableIdentifier, LockUsage.CLEAN_FILES_LOCK, errorMsg)
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(carbonTable, true)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = true, carbonTable, currentTablePartitions.map(_.asJava).orNull)
CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, true)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
index ad624ee..578138f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonColumnValidator.scala
@@ -16,12 +16,12 @@
*/
package org.apache.carbondata.spark
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
- /**
- * Carbon column validator
- */
+/**
+ * Carbon column validator
+ */
class CarbonColumnValidator extends ColumnValidator {
def validateColumns(allColumns: Seq[ColumnSchema]) {
allColumns.foreach { columnSchema =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
index 2fdd2b1..e69de29 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.load
-
-import java.text.SimpleDateFormat
-
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.processing.loading.sort.SortScopeOptions
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-
-object ValidateUtil {
-
- /**
- * validates both timestamp and date for illegal values
- *
- * @param dateTimeLoadFormat
- * @param dateTimeLoadOption
- */
- def validateDateTimeFormat(dateTimeLoadFormat: String, dateTimeLoadOption: String): Unit = {
- // allowing empty value to be configured for dateformat option.
- if (dateTimeLoadFormat != null && dateTimeLoadFormat.trim != "") {
- try {
- new SimpleDateFormat(dateTimeLoadFormat)
- } catch {
- case _: IllegalArgumentException =>
- throw new MalformedCarbonCommandException(s"Error: Wrong option: $dateTimeLoadFormat is" +
- s" provided for option $dateTimeLoadOption")
- }
- }
- }
-
- def validateSortScope(carbonTable: CarbonTable, sortScope: String): Unit = {
- if (sortScope != null) {
- // Don't support use global sort on partitioned table.
- if (carbonTable.getPartitionInfo(carbonTable.getTableName) != null &&
- !carbonTable.isHivePartitionTable &&
- sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString)) {
- throw new MalformedCarbonCommandException("Don't support use global sort on partitioned " +
- "table.")
- }
- }
- }
-
- def validateGlobalSortPartitions(globalSortPartitions: String): Unit = {
- if (globalSortPartitions != null) {
- try {
- val num = globalSortPartitions.toInt
- if (num <= 0) {
- throw new MalformedCarbonCommandException("'GLOBAL_SORT_PARTITIONS' should be greater " +
- "than 0.")
- }
- } catch {
- case e: NumberFormatException => throw new MalformedCarbonCommandException(e.getMessage)
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 3fbed6f..444c9dc 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -299,11 +299,11 @@ class CarbonMergerRDD[K, V](
val splits = format.getSplits(job)
// keep on assigning till last one is reached.
- if (null != splits && splits.size > 0) splitsOfLastSegment =
- splits.asScala
+ if (null != splits && splits.size > 0) {
+ splitsOfLastSegment = splits.asScala
.map(_.asInstanceOf[CarbonInputSplit])
.filter { split => FileFormat.COLUMNAR_V3.equals(split.getFileFormat) }.toList.asJava
-
+ }
carbonInputSplits ++:= splits.asScala.map(_.asInstanceOf[CarbonInputSplit]).filter{ entry =>
val blockInfo = new TableBlockInfo(entry.getPath.toString,
entry.getStart, entry.getSegmentId,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 7815c99..9104a32 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.types.{MetadataBuilder, StringType}
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.FileUtils
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -55,7 +56,7 @@ import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
object CommonUtil {
@@ -633,13 +634,6 @@ object CommonUtil {
parsedPropertyValueString
}
-
- def readLoadMetadataDetails(model: CarbonLoadModel): Unit = {
- val metadataPath = model.getCarbonDataLoadSchema.getCarbonTable.getMetadataPath
- val details = SegmentStatusManager.readLoadMetadata(metadataPath)
- model.setLoadMetadataDetails(new util.ArrayList[LoadMetadataDetails](details.toList.asJava))
- }
-
def configureCSVInputFormat(configuration: Configuration,
carbonLoadModel: CarbonLoadModel): Unit = {
CSVInputFormat.setCommentCharacter(configuration, carbonLoadModel.getCommentChar)
@@ -681,65 +675,6 @@ object CommonUtil {
}
}
- def getCsvHeaderColumns(
- carbonLoadModel: CarbonLoadModel,
- hadoopConf: Configuration,
- staticPartitionCols: util.List[String] = new util.ArrayList[String]()): Array[String] = {
- val delimiter = if (StringUtils.isEmpty(carbonLoadModel.getCsvDelimiter)) {
- CarbonCommonConstants.COMMA
- } else {
- CarbonUtil.delimiterConverter(carbonLoadModel.getCsvDelimiter)
- }
- var csvFile: String = null
- var csvHeader: String = carbonLoadModel.getCsvHeader
- var csvColumns = if (StringUtils.isBlank(csvHeader)) {
- // read header from csv file
- csvFile = carbonLoadModel.getFactFilePath.split(",")(0)
- csvHeader = CarbonUtil.readHeader(csvFile, hadoopConf)
- if (StringUtils.isBlank(csvHeader)) {
- throw new CarbonDataLoadingException("First line of the csv is not valid.")
- }
- csvHeader.toLowerCase().split(delimiter).map(_.replaceAll("\"", "").trim)
- } else {
- csvHeader.toLowerCase.split(CarbonCommonConstants.COMMA).map(_.trim)
- }
-
- if (!CarbonDataProcessorUtil.isHeaderValid(carbonLoadModel.getTableName, csvColumns,
- carbonLoadModel.getCarbonDataLoadSchema, staticPartitionCols)) {
- if (csvFile == null) {
- LOGGER.error("CSV header in DDL is not proper."
- + " Column names in schema and CSV header are not the same.")
- throw new CarbonDataLoadingException(
- "CSV header in DDL is not proper. Column names in schema and CSV header are "
- + "not the same.")
- } else {
- LOGGER.error(
- "CSV header in input file is not proper. Column names in schema and csv header are not "
- + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile))
- throw new CarbonDataLoadingException(
- "CSV header in input file is not proper. Column names in schema and csv header are not "
- + "the same. Input file : " + CarbonUtil.removeAKSK(csvFile))
- }
- }
- // In case of static partition columns just change the name of header if already exists as
- // we should not take the column from csv file and add them as new columns at the end.
- if (staticPartitionCols.size() > 0) {
- val scalaIgnoreColumns = staticPartitionCols.asScala
- var updatedCols = csvColumns.map{col =>
- if (scalaIgnoreColumns.exists(_.equalsIgnoreCase(col))) {
- col + "1"
- } else {
- col
- }
- }.toList.asJava
- updatedCols = new util.ArrayList[String](updatedCols)
- updatedCols.addAll(staticPartitionCols)
- updatedCols.asScala.toArray
- } else {
- csvColumns
- }
- }
-
def validateMaxColumns(csvHeaders: Array[String], maxColumns: String): Int = {
/*
User configures both csvheadercolumns, maxcolumns,
@@ -880,6 +815,7 @@ object CommonUtil {
try {
val carbonTable = CarbonMetadata.getInstance
.getCarbonTable(identifier.getCarbonTableIdentifier.getTableUniqueName)
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(carbonTable, true)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = true, carbonTable, null)
} catch {
[07/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index de97e82..540607d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -58,7 +58,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,20 +68,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
- protected List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
protected SegmentProperties segmentProperties;
/**
* it has index at which given dimension is stored in file
*/
- protected int[] dimensionBlocksIndex;
+ int[] dimensionChunkIndex;
/**
* it has index at which given measure is stored in file
*/
- protected int[] measureBlocksIndex;
+ int[] measureChunkIndex;
private Map<Integer, GenericQueryType> complexDimensionInfoMap;
@@ -89,18 +89,18 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* flag to check whether the filter dimension is present in current block list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean[] isDimensionPresentInCurrentBlock;
+ boolean[] isDimensionPresentInCurrentBlock;
/**
* flag to check whether the filter measure is present in current block list of measures.
* Applicable for restructure scenarios
*/
- protected boolean[] isMeasurePresentInCurrentBlock;
+ boolean[] isMeasurePresentInCurrentBlock;
/**
* is dimension column data is natural sorted
*/
- protected boolean isNaturalSorted;
+ boolean isNaturalSorted;
/**
* date direct dictionary generator
@@ -124,10 +124,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
if (this.dimColEvaluatorInfoList.size() > 0) {
this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
- this.dimensionBlocksIndex = new int[]{0};
+ this.dimensionChunkIndex = new int[]{0};
}
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
@@ -136,10 +136,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
if (this.msrColEvalutorInfoList.size() > 0) {
this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
- this.measureBlocksIndex = new int[] {0};
+ this.measureChunkIndex = new int[] {0};
}
this.exp = exp;
this.tableIdentifier = tableIdentifier;
@@ -148,22 +148,22 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.DATE);
this.timestampDictionaryGenerator =
DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.TIMESTAMP);
- initDimensionBlockIndexes();
- initMeasureBlockIndexes();
+ initDimensionChunkIndexes();
+ initMeasureChunkIndexes();
}
/**
* This method will initialize the dimension info for the current block to be
* used for filtering the data
*/
- private void initDimensionBlockIndexes() {
+ private void initDimensionChunkIndexes() {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
// find the dimension in the current block dimensions list
CarbonDimension dimensionFromCurrentBlock = segmentProperties
.getDimensionFromCurrentBlock(dimColEvaluatorInfoList.get(i).getDimension());
if (null != dimensionFromCurrentBlock) {
dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- this.dimensionBlocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+ this.dimensionChunkIndex[i] = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimensionFromCurrentBlock.getOrdinal());
isDimensionPresentInCurrentBlock[i] = true;
}
@@ -174,14 +174,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* This method will initialize the measure info for the current block to be
* used for filtering the data
*/
- private void initMeasureBlockIndexes() {
+ private void initMeasureChunkIndexes() {
for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
// find the measure in the current block measures list
CarbonMeasure measureFromCurrentBlock = segmentProperties.getMeasureFromCurrentBlock(
msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId());
if (null != measureFromCurrentBlock) {
msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal());
- this.measureBlocksIndex[i] = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ this.measureChunkIndex[i] = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(measureFromCurrentBlock.getOrdinal());
isMeasurePresentInCurrentBlock[i] = true;
}
@@ -189,9 +189,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- readBlocks(blockChunkHolder);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
int[] numberOfRows = null;
@@ -199,31 +199,31 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
if (dimColEvaluatorInfoList.size() > 0) {
if (isDimensionPresentInCurrentBlock[0]) {
- pageNumbers =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount();
- numberOfRows =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount();
+ pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
+ .getPagesCount();
+ numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
+ .getRowCount();
} else {
// specific for restructure case where default values need to be filled
- pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
+ pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
- numberOfRows[i] = blockChunkHolder.getDataBlock().getPageRowCount(i);
+ numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
if (msrColEvalutorInfoList.size() > 0) {
if (isMeasurePresentInCurrentBlock[0]) {
- pageNumbers =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount();
- numberOfRows =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount();
+ pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .getPagesCount();
+ numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .getRowCount();
} else {
// specific for restructure case where default values need to be filled
- pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
+ pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
- numberOfRows[i] = blockChunkHolder.getDataBlock().getPageRowCount(i);
+ numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
@@ -234,10 +234,12 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
BitSet prvBitset = null;
// if bitset pipe line is enabled then use rowid from previous bitset
// otherwise use older flow
- if (!useBitsetPipeLine || null == blockChunkHolder.getBitSetGroup() || null == bitSetGroup
- .getBitSet(i) || blockChunkHolder.getBitSetGroup().getBitSet(i).isEmpty()) {
+ if (!useBitsetPipeLine ||
+ null == rawBlockletColumnChunks.getBitSetGroup() ||
+ null == bitSetGroup.getBitSet(i) ||
+ rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
for (int index = 0; index < numberOfRows[i]; index++) {
- createRow(blockChunkHolder, row, i, index);
+ createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
@@ -253,10 +255,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
}
} else {
- prvBitset = blockChunkHolder.getBitSetGroup().getBitSet(i);
+ prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
for (int index = prvBitset.nextSetBit(0);
index >= 0; index = prvBitset.nextSetBit(index + 1)) {
- createRow(blockChunkHolder, row, i, index);
+ createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
@@ -371,8 +373,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param index
* @throws IOException
*/
- private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index)
- throws IOException {
+ private void createRow(RawBlockletColumnChunks blockChunkHolder, RowIntf row, int pageIndex,
+ int index) throws IOException {
Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
String memberString;
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
@@ -389,14 +391,14 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
record[dimColumnEvaluatorInfo.getRowIndex()] =
dimColumnEvaluatorInfo.getDimension().getDefaultValue();
}
- DimensionColumnDataChunk columnDataChunk =
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]
- .convertToDimColDataChunk(pageIndex);
+ DimensionColumnPage columnDataChunk =
+ blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]
+ .decodeColumnPage(pageIndex);
if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
+ && columnDataChunk instanceof VariableLengthDimensionColumnPage) {
- VariableLengthDimensionDataChunk dimensionColumnDataChunk =
- (VariableLengthDimensionDataChunk) columnDataChunk;
+ VariableLengthDimensionColumnPage dimensionColumnDataChunk =
+ (VariableLengthDimensionColumnPage) columnDataChunk;
byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
if (null != memberBytes) {
if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
@@ -409,8 +411,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
dimColumnEvaluatorInfo.getDimension().getDataType());
}
} else {
- int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex,
- dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
+ int dictionaryValue = readSurrogatesFromColumnChunk(blockChunkHolder, index, pageIndex,
+ dimColumnEvaluatorInfo, dimensionChunkIndex[i]);
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
@@ -428,11 +430,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
try {
- GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
complexType.parseBlocksAndReturnComplexColumnByteArray(
- blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
+ blockChunkHolder.getDimensionRawColumnChunks(), index, pageIndex, dataOutputStream);
record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
byteStream.close();
@@ -471,8 +473,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
Object msrValue;
ColumnPage columnPage =
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
- .convertToColumnPage(pageIndex);
+ blockChunkHolder.getMeasureRawColumnChunks()[measureChunkIndex[0]]
+ .decodeColumnPage(pageIndex);
if (msrType == DataTypes.BOOLEAN) {
msrValue = columnPage.getBoolean(index);
} else if (msrType == DataTypes.SHORT) {
@@ -574,10 +576,10 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param dimColumnEvaluatorInfo
* @return
*/
- private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index, int page,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
- DimensionColumnDataChunk dataChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex].convertToDimColDataChunk(page);
+ private int readSurrogatesFromColumnChunk(RawBlockletColumnChunks blockChunkHolder, int index,
+ int page, DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int chunkIndex) {
+ DimensionColumnPage dataChunk =
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex].decodeColumnPage(page);
if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
byte[] rawData = dataChunk.getChunkData(index);
ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
@@ -593,7 +595,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param dimColumnEvaluatorInfo
* @return read surrogate of given row of given column group dimension
*/
- private int readSurrogatesFromColumnGroupBlock(DimensionColumnDataChunk chunk, int index,
+ private int readSurrogatesFromColumnGroupBlock(DimensionColumnPage chunk, int index,
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
try {
KeyStructureInfo keyStructureInfo =
@@ -601,9 +603,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
byte[] colData = chunk.getChunkData(index);
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
- QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
+ QueryUtil.getColumnGroupId(segmentProperties, dimensionChunkIndex[0]);
return (int) result[segmentProperties
- .getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
+ .getColumnGroupMdKeyOrdinal(colGroupId, dimensionChunkIndex[0])];
} catch (KeyGenException e) {
LOGGER.error(e);
}
@@ -611,33 +613,36 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.set(0);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
- if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
- blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
- blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]])
+ {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[i]] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), dimensionChunkIndex[i]);
}
} else {
- GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
- complexType.fillRequiredBlockData(blockChunkHolder);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionChunkIndex[i]);
+ complexType.fillRequiredBlockData(rawBlockletColumnChunks);
}
}
if (null != msrColEvalutorInfoList) {
for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) {
- blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] =
- blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 3981211..1f63a81 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -32,12 +32,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -56,12 +55,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
*/
private boolean isDefaultValuePresentInFilter;
private int lastDimensionColOrdinal = 0;
- public RowLevelRangeGrtThanFiterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+
+ RowLevelRangeGrtThanFiterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- Object[] msrFilterRangeValues,
- SegmentProperties segmentProperties) {
+ Object[] msrFilterRangeValues, SegmentProperties segmentProperties) {
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
@@ -111,17 +109,18 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
boolean isScanRequired = false;
byte[] maxValue = null;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ maxValue = blockMaxValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(maxValue, filterRangeValues);
}
} else {
@@ -168,24 +167,25 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -197,26 +197,28 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -226,7 +228,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i],
msrColEvalutorInfoList.get(0).getType()));
ColumnPage columnPage =
- rawColumnChunk.convertToColumnPage(i);
+ rawColumnChunk.decodeColumnPage(i);
if (compare < 0 && columnPage.getNullBits().isEmpty()) {
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
@@ -239,7 +241,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -296,16 +298,16 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -317,12 +319,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int last = 0;
@@ -330,11 +332,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
byte[][] filterValues = this.filterRangeValues;
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[i],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[i],
numerOfRows);
}
// Logic will handle the case where the range filter member is not present in block
@@ -350,7 +352,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
// index needs to be compared by the filter member if its > filter then from that
// index the bitset will be considered for filtering process.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
> 0) {
start = start + 1;
}
@@ -358,7 +360,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
last = start;
for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last++;
}
startIndex = last;
@@ -376,11 +378,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -391,11 +393,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
int startIndex = 0;
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], true);
if (start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[k],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[k],
numerOfRows);
}
if (start < 0) {
@@ -406,7 +408,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its > filter then from that
// index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) > 0) {
start = start + 1;
}
}
@@ -423,7 +425,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) > 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) > 0) {
bitSet.set(i);
}
}
@@ -432,21 +434,24 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index f2ddcb6..9140a11 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -32,12 +32,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -47,7 +46,7 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
- protected byte[][] filterRangeValues;
+ private byte[][] filterRangeValues;
private Object[] msrFilterRangeValues;
private SerializableComparator comparator;
/**
@@ -56,7 +55,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
private boolean isDefaultValuePresentInFilter;
private int lastDimensionColOrdinal = 0;
- public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
+ RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
@@ -116,11 +115,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
byte[] maxValue = null;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ maxValue = blockMaxValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ maxValue = blockMaxValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(maxValue, filterRangeValues);
}
} else {
@@ -166,25 +165,26 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -196,26 +196,28 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
- } else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ } else {
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
@@ -225,28 +227,27 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i],
msrColEvalutorInfoList.get(0).getType()));
ColumnPage columnPage =
- rawColumnChunk.convertToColumnPage(i);
+ rawColumnChunk.decodeColumnPage(i);
if (compare <= 0 && columnPage.getNullBits().isEmpty()) {
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
}
- return null;
}
@Override
@@ -297,16 +298,16 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -318,12 +319,12 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int last = 0;
@@ -331,7 +332,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
byte[][] filterValues = this.filterRangeValues;
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], false);
if (start < 0) {
start = -(start + 1);
@@ -342,14 +343,14 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
> 0) {
start = start + 1;
}
}
last = start;
for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last++;
}
startIndex = last;
@@ -366,11 +367,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -381,7 +382,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
int startIndex = 0;
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], false);
if (start < 0) {
start = -(start + 1);
@@ -391,7 +392,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) > 0) {
start = start + 1;
}
}
@@ -409,7 +410,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) >= 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) >= 0) {
bitSet.set(i);
}
}
@@ -418,21 +419,24 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index a44bc1a..120671f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,12 +35,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -118,11 +117,11 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- minValue = blockMinValue[dimensionBlocksIndex[0]];
+ minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -168,60 +167,63 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -278,7 +280,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -287,7 +289,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+ segmentProperties.getDimensions().get(dimensionChunkIndex[0]);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -298,14 +300,14 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows,
defaultValue);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows, defaultValue);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -317,12 +319,12 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -333,7 +335,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -350,7 +352,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start < 0) {
start = -(start + 1);
@@ -361,14 +363,14 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0) {
start = start - 1;
}
}
last = start;
for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last--;
}
startIndex = last;
@@ -385,12 +387,12 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @param defaultValue
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -403,7 +405,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -420,7 +422,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], true);
if (start < 0) {
start = -(start + 1);
@@ -430,7 +432,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
// When negative value of start is returned from getFirstIndexUsingBinarySearch the Start
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) < 0) {
start = start - 1;
}
}
@@ -447,7 +449,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) <= 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) <= 0) {
bitSet.set(i);
}
}
@@ -456,21 +458,24 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
[48/54] [abbrv] carbondata git commit: [CARBONDATA-2213][DataMap]
Fixed wrong version for module datamap-example
Posted by ra...@apache.org.
[CARBONDATA-2213][DataMap] Fixed wrong version for module datamap-example
The version of Module ‘carbondata-datamap-example’ should be 1.4.0-snapshot instead of 1.3.0-snapshot, otherwise compilation will failed.
This closes #2011
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5397c05c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5397c05c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5397c05c
Branch: refs/heads/master
Commit: 5397c05c9bd80a0ee956046d027b6a2900475f52
Parents: 5eb476f
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Feb 28 10:31:02 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
datamap/examples/pom.xml | 8 ++++----
.../datamap/examples/MinMaxIndexDataMapFactory.java | 1 -
.../datamap/lucene/LuceneCoarseGrainDataMap.java | 3 ++-
.../lucene/LuceneCoarseGrainDataMapFactory.java | 9 ++++++---
.../datamap/lucene/LuceneDataMapFactoryBase.java | 16 +++++++++++-----
.../datamap/lucene/LuceneDataMapWriter.java | 5 +++--
.../datamap/lucene/LuceneFineGrainDataMap.java | 3 ++-
.../lucene/LuceneFineGrainDataMapFactory.java | 9 ++++++---
8 files changed, 34 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/examples/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/examples/pom.xml b/datamap/examples/pom.xml
index 8539a86..30e1522 100644
--- a/datamap/examples/pom.xml
+++ b/datamap/examples/pom.xml
@@ -22,10 +22,10 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.carbondata</groupId>
- <artifactId>carbondata-parent</artifactId>
- <version>1.4.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>carbondata-datamap-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
index 9fea55b..45dee2a 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
index 0b7df86..580f18b 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMap.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -133,7 +134,7 @@ public class LuceneCoarseGrainDataMap extends CoarseGrainDataMap {
*/
@Override
public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<String> partitions) throws IOException {
+ List<PartitionSpec> partitions) throws IOException {
// convert filter expr into lucene list query
List<String> fields = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
index 7f9cc1c..04160c0 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneCoarseGrainDataMapFactory.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
import org.apache.carbondata.core.memory.MemoryException;
@@ -41,13 +42,14 @@ public class LuceneCoarseGrainDataMapFactory extends LuceneDataMapFactoryBase<Co
/**
* Get the datamap for segmentid
*/
- public List<CoarseGrainDataMap> getDataMaps(String segmentId) throws IOException {
+ @Override
+ public List<CoarseGrainDataMap> getDataMaps(Segment segment) throws IOException {
List<CoarseGrainDataMap> lstDataMap = new ArrayList<>();
CoarseGrainDataMap dataMap = new LuceneCoarseGrainDataMap(analyzer);
try {
dataMap.init(new DataMapModel(
LuceneDataMapWriter.genDataMapStorePath(
- tableIdentifier.getTablePath(), segmentId, dataMapName)));
+ tableIdentifier.getTablePath(), segment.getSegmentNo(), dataMapName)));
} catch (MemoryException e) {
LOGGER.error("failed to get lucene datamap , detail is {}" + e.getMessage());
return lstDataMap;
@@ -59,9 +61,10 @@ public class LuceneCoarseGrainDataMapFactory extends LuceneDataMapFactoryBase<Co
/**
* Get datamaps for distributable object.
*/
+ @Override
public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
throws IOException {
- return getDataMaps(distributable.getSegmentId());
+ return getDataMaps(distributable.getSegment());
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 5eb7054..88e70d5 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
@@ -136,23 +137,26 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFac
/**
* Return a new write for this datamap
*/
- public DataMapWriter createWriter(String segmentId, String writeDirectoryPath) {
+ @Override
+ public DataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
LOGGER.info("lucene data write to " + writeDirectoryPath);
return new LuceneDataMapWriter(
- tableIdentifier, dataMapName, segmentId, writeDirectoryPath, true);
+ tableIdentifier, dataMapName, segment, writeDirectoryPath, true);
}
/**
* Get all distributable objects of a segmentid
*/
- public List<DataMapDistributable> toDistributable(String segmentId) {
+ @Override
+ public List<DataMapDistributable> toDistributable(Segment segment) {
List<DataMapDistributable> lstDataMapDistribute = new ArrayList<DataMapDistributable>();
DataMapDistributable luceneDataMapDistributable = new LuceneDataMapDistributable(
- CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segmentId));
+ CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segment.getSegmentNo()));
lstDataMapDistribute.add(luceneDataMapDistributable);
return lstDataMapDistribute;
}
+ @Override
public void fireEvent(Event event) {
}
@@ -160,13 +164,15 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFac
/**
* Clears datamap of the segment
*/
- public void clear(String segmentId) {
+ @Override
+ public void clear(Segment segment) {
}
/**
* Clear all datamaps from memory
*/
+ @Override
public void clear() {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
index 849fc2e..86b2382 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
@@ -24,6 +24,7 @@ import java.io.UnsupportedEncodingException;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -84,9 +85,9 @@ public class LuceneDataMapWriter extends DataMapWriter {
private static final String ROWID_NAME = "rowId";
- LuceneDataMapWriter(AbsoluteTableIdentifier identifier, String dataMapName, String segmentId,
+ LuceneDataMapWriter(AbsoluteTableIdentifier identifier, String dataMapName, Segment segment,
String writeDirectoryPath, boolean isFineGrain) {
- super(identifier, segmentId, writeDirectoryPath);
+ super(identifier, segment, writeDirectoryPath);
this.dataMapName = dataMapName;
this.isFineGrain = isFineGrain;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
index c649545..faa4cbe 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainBlocklet;
import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
@@ -144,7 +145,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap {
*/
@Override
public List<FineGrainBlocklet> prune(FilterResolverIntf filterExp,
- SegmentProperties segmentProperties, List<String> partitions) throws IOException {
+ SegmentProperties segmentProperties, List<PartitionSpec> partitions) throws IOException {
// convert filter expr into lucene list query
List<String> fields = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5397c05c/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
index e35d5bf..a9376bc 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapFactory.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
import org.apache.carbondata.core.datamap.dev.fgdatamap.FineGrainDataMap;
import org.apache.carbondata.core.memory.MemoryException;
@@ -37,13 +38,14 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine
/**
* Get the datamap for segmentid
*/
- public List<FineGrainDataMap> getDataMaps(String segmentId) throws IOException {
+ @Override
+ public List<FineGrainDataMap> getDataMaps(Segment segment) throws IOException {
List<FineGrainDataMap> lstDataMap = new ArrayList<>();
FineGrainDataMap dataMap = new LuceneFineGrainDataMap(analyzer);
try {
dataMap.init(new DataMapModel(
LuceneDataMapWriter.genDataMapStorePath(
- tableIdentifier.getTablePath(), segmentId, dataMapName)));
+ tableIdentifier.getTablePath(), segment.getSegmentNo(), dataMapName)));
} catch (MemoryException e) {
LOGGER.error("failed to get lucene datamap , detail is {}" + e.getMessage());
return lstDataMap;
@@ -55,9 +57,10 @@ public class LuceneFineGrainDataMapFactory extends LuceneDataMapFactoryBase<Fine
/**
* Get datamaps for distributable object.
*/
+ @Override
public List<FineGrainDataMap> getDataMaps(DataMapDistributable distributable)
throws IOException {
- return getDataMaps(distributable.getSegmentId());
+ return getDataMaps(distributable.getSegment());
}
@Override
[31/54] [abbrv] carbondata git commit: [CARBONDATA-2216][Test] Fix
bugs in sdv tests
Posted by ra...@apache.org.
[CARBONDATA-2216][Test] Fix bugs in sdv tests
This closes #2012
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f9291cdb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f9291cdb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f9291cdb
Branch: refs/heads/master
Commit: f9291cdb3b480d3ad46f963b5c772b8fa3185034
Parents: 5397c05
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Feb 28 16:02:55 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../core/datamap/dev/expr/AndDataMapExprWrapper.java | 2 +-
.../core/datamap/dev/expr/DataMapExprWrapper.java | 2 +-
.../datamap/dev/expr/DataMapExprWrapperImpl.java | 2 +-
.../core/datamap/dev/expr/OrDataMapExprWrapper.java | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 3 ++-
.../cluster/sdv/generated/MergeIndexTestCase.scala | 15 +++++++++++++--
.../preaaggregate/PreAggregateTableHelper.scala | 5 -----
.../apache/spark/sql/optimizer/CarbonFilters.scala | 4 +---
8 files changed, 20 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
index 12b60b4..74469d7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
index ddb19e9..14cfc33 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.List;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
index d4be416..c6b011c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
@@ -22,8 +22,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
index a8bfcad..37cd5dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
@@ -22,8 +22,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.DataMapLevel;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 81b5fde..bcc487e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -929,7 +929,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
TableDataMap blockletMap = DataMapStoreManager.getInstance().getDefaultDataMap(identifier);
LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager
.readTableStatusFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()));
- SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier, loadMetadataDetails);
+ SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(
+ identifier, loadMetadataDetails);
SegmentStatusManager.ValidAndInvalidSegmentsInfo allSegments =
new SegmentStatusManager(identifier).getValidAndInvalidSegments(loadMetadataDetails);
Map<String, Long> blockRowCountMapping = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
index c0abe4e..b886b11 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/MergeIndexTestCase.scala
@@ -92,18 +92,29 @@ class MergeIndexTestCase extends QueryTest with BeforeAndAfterAll {
val table = CarbonMetadata.getInstance().getCarbonTable("default","carbon_automation_nonmerge")
new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 0)
+ assert(getMergedIndexFileCount("default", "carbon_automation_nonmerge", "0.1") == 1)
checkAnswer(sql("""Select count(*) from carbon_automation_nonmerge"""), rows)
}
private def getIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
+ getFileCount(dbName, tableName, segment, CarbonTablePath.INDEX_FILE_EXT)
+ }
+
+ private def getMergedIndexFileCount(dbName: String, tableName: String, segment: String): Int = {
+ getFileCount(dbName, tableName, segment, CarbonTablePath.MERGE_INDEX_FILE_EXT)
+ }
+
+ private def getFileCount(dbName: String,
+ tableName: String,
+ segment: String,
+ suffix: String): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(dbName, tableName)
val identifier = carbonTable.getAbsoluteTableIdentifier
val path = CarbonTablePath
.getSegmentPath(identifier.getTablePath, segment)
val carbonFiles = FileFactory.getCarbonFile(path).listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = {
- file.getName.endsWith(CarbonTablePath
- .INDEX_FILE_EXT)
+ file.getName.endsWith(suffix)
}
})
if (carbonFiles != null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
index 4c0e637..1f1e1e6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateTableHelper.scala
@@ -26,16 +26,11 @@ import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand
import org.apache.spark.sql.execution.command.table.CarbonCreateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
-import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
-import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
-import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchema, CarbonTable, DataMapSchema}
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f9291cdb/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index 2213fdb..3344449 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -40,11 +40,9 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression, MatchExpression}
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
-import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.util.ThreadLocalSessionInfo
-import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.ThreadLocalSessionInfo
import org.apache.carbondata.datamap.TextMatch
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil
[49/54] [abbrv] carbondata git commit: [HOTFIX] Add dava doc for
datamap interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
deleted file mode 100644
index a05a8c2..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestIndexDataMapCommand.scala
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.datamap
-
-import java.io.{File, FilenameFilter}
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.test.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.common.exceptions.MetadataProcessException
-import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.CarbonTablePath
-
-class TestIndexDataMapCommand extends QueryTest with BeforeAndAfterAll {
-
- val testData = s"$resourcesPath/sample.csv"
-
- override def beforeAll {
- sql("drop table if exists datamaptest")
- sql("drop table if exists datamapshowtest")
- sql("drop table if exists uniqdata")
- sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'")
- }
-
- val newClass = "org.apache.spark.sql.CarbonSource"
-
- test("test datamap create: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
- }
- }
-
- test("test datamap create with dmproperties: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- }
- }
-
- test("test datamap create with existing name: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql(
- s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- }
- }
-
- test("test datamap create with preagg") {
- sql("drop datamap if exists datamap3 on table datamaptest")
- sql(
- "create datamap datamap3 on table datamaptest using 'preaggregate' as select count(a) from datamaptest")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 1)
- assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap3"))
- assert(dataMapSchemaList.get(0).getChildSchema.getTableName.equals("datamaptest_datamap3"))
- }
-
- test("check hivemetastore after drop datamap") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- "true")
- sql("drop table if exists hiveMetaStoreTable")
- sql("create table hiveMetaStoreTable (a string, b string, c string) stored by 'carbondata'")
-
- sql(
- "create datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable using 'preaggregate' as select count(a) from hiveMetaStoreTable")
- checkExistence(sql("show datamap on table hiveMetaStoreTable"), true, "datamap_hiveMetaStoreTable")
-
- sql("drop datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
- checkExistence(sql("show datamap on table hiveMetaStoreTable"), false, "datamap_hiveMetaStoreTable")
-
- } finally {
- sql("drop table hiveMetaStoreTable")
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("drop the table having pre-aggregate") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- "true")
- sql("drop table if exists hiveMetaStoreTable_1")
- sql("create table hiveMetaStoreTable_1 (a string, b string, c string) stored by 'carbondata'")
-
- sql(
- "create datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1 using 'preaggregate' as select count(a) from hiveMetaStoreTable_1")
-
- checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
- true,
- "datamap_hiveMetaStoreTable_1")
-
- sql("drop datamap datamap_hiveMetaStoreTable_1 on table hiveMetaStoreTable_1")
- checkExistence(sql("show datamap on table hiveMetaStoreTable_1"),
- false,
- "datamap_hiveMetaStoreTable_1")
- assert(sql("show datamap on table hiveMetaStoreTable_1").collect().length == 0)
- sql("drop table hiveMetaStoreTable_1")
-
- checkExistence(sql("show tables"), false, "datamap_hiveMetaStoreTable_1")
- }
- finally {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("test datamap create with preagg with duplicate name") {
- sql(
- s"""
- | CREATE DATAMAP datamap10 ON TABLE datamaptest
- | USING 'preaggregate'
- | AS SELECT COUNT(a) FROM datamaptest
- """.stripMargin)
- intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP datamap10 ON TABLE datamaptest
- | USING 'preaggregate'
- | AS SELECT COUNT(a) FROM datamaptest
- """.stripMargin)
- }
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- }
-
- test("test drop non-exist datamap") {
- intercept[NoSuchDataMapException] {
- sql("drop datamap nonexist on table datamaptest")
- }
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- }
-
- test("test show datamap without preaggregate: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' ")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
- checkExistence(sql("SHOW DATAMAP ON TABLE datamapshowtest"), true, "datamap1", "datamap2", "(NA)", newClass)
- }
- }
-
- test("test show datamap with preaggregate: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 2)
- checkExistence(frame, true, "datamap1", "datamap2", "(NA)", newClass, "default.datamapshowtest_datamap1")
- }
- }
-
- test("test show datamap with no datamap") {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- assert(sql("show datamap on table datamapshowtest").collect().length == 0)
- }
-
- test("test show datamap after dropping datamap: don't support using non-exist class") {
- intercept[MetadataProcessException] {
- sql("drop table if exists datamapshowtest")
- sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
- sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
- sql(s"CREATE DATAMAP datamap2 ON TABLE datamapshowtest USING '$newClass' ")
- sql("drop datamap datamap1 on table datamapshowtest")
- val frame = sql("show datamap on table datamapshowtest")
- assert(frame.collect().length == 1)
- checkExistence(frame, true, "datamap2", "(NA)", newClass)
- }
- }
-
- test("test if preaggregate load is successfull for hivemetastore") {
- try {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("DROP TABLE IF EXISTS maintable")
- sql(
- """
- | CREATE TABLE maintable(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- """.stripMargin)
- sql(
- s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age) from maintable group by id"""
-
- .stripMargin)
- sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
- checkAnswer(sql(s"select * from maintable_preagg_sum"),
- Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
- } finally {
- CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- }
- }
-
- test("test preaggregate load for decimal column for hivemetastore") {
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true")
- sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
- sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1")
- sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1")
- checkAnswer(sql("select * from uniqdata_uniqdata_agg"), Seq(Row(12345678901.1234000000, 12345678901.1234000000)))
- sql("drop datamap if exists uniqdata_agg on table uniqdata")
- }
-
- test("create pre-agg table with path") {
- sql("drop table if exists main_preagg")
- sql("drop table if exists main ")
- val warehouse = s"$metastoredb/warehouse"
- val path = warehouse + "/" + System.nanoTime + "_preAggTestPath"
- sql(
- s"""
- | create table main(
- | year int,
- | month int,
- | name string,
- | salary int)
- | stored by 'carbondata'
- | tblproperties('sort_columns'='month,year,name')
- """.stripMargin)
- sql("insert into main select 10,11,'amy',12")
- sql("insert into main select 10,11,'amy',14")
- sql(
- s"""
- | create datamap preagg
- | on table main
- | using 'preaggregate'
- | dmproperties ('path'='$path')
- | as select name,avg(salary)
- | from main
- | group by name
- """.stripMargin)
- assertResult(true)(new File(path).exists())
- assertResult(true)(new File(s"${CarbonTablePath.getSegmentPath(path, "0")}")
- .list(new FilenameFilter {
- override def accept(dir: File, name: String): Boolean = {
- name.contains(CarbonCommonConstants.FACT_FILE_EXT)
- }
- }).length > 0)
- checkAnswer(sql("select name,avg(salary) from main group by name"), Row("amy", 13.0))
- checkAnswer(sql("select * from main_preagg"), Row("amy", 26, 2))
- sql("drop datamap preagg on table main")
- assertResult(false)(new File(path).exists())
- sql("drop table main")
- }
-
- override def afterAll {
- sql("DROP TABLE IF EXISTS maintable")
- sql("drop table if exists uniqdata")
- CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
- CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
- sql("drop table if exists datamaptest")
- sql("drop table if exists datamapshowtest")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 17beace..13db652 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -29,6 +29,9 @@ import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandExcepti
class TestSortColumns extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+
SparkUtil4Test.createTaskMockUp(sqlContext)
dropTable
CarbonProperties.getInstance()
@@ -380,6 +383,8 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
override def afterAll = {
dropTable
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
}
def dropTable = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
index e98b701..2cfa4c2 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
@@ -16,21 +16,29 @@
*/
package org.apache.carbondata.spark.testsuite.sortcolumns
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.test.util.QueryTest
-class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll {
+class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
override def beforeAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, "true")
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
dropTable
sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
}
+ override def beforeEach(): Unit = {
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+
test("create table with no dictionary sort_columns") {
sql("CREATE TABLE sorttable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
@@ -249,6 +257,9 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE)
+ CarbonProperties.getInstance().addProperty(
+ CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+
dropTable
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
index a71e0d8..10955a3 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapProvider.java
@@ -18,7 +18,6 @@
package org.apache.carbondata.datamap;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
@@ -34,7 +33,7 @@ import org.apache.spark.sql.SparkSession;
* <ol>
* <li> MVDataMap: materialized view type of DataMap to accelerate olap style query,
* like SPJG query (select, predicate, join, groupby) </li>
- * <li> IndexDataMap: index type of DataMap to accelerate filter query </li>
+ * <li> DataMap: index type of DataMap to accelerate filter query </li>
* </ol>
*
* <p>
@@ -47,15 +46,14 @@ import org.apache.spark.sql.SparkSession;
* <li> preaggregate: one type of MVDataMap that do pre-aggregate of single table </li>
* <li> timeseries: one type of MVDataMap that do pre-aggregate based on time dimension
* of the table </li>
- * <li> class name of {@link org.apache.carbondata.core.datamap.dev.IndexDataMapFactory}
- * implementation: Developer can implement new type of IndexDataMap by extending
- * {@link org.apache.carbondata.core.datamap.dev.IndexDataMapFactory} </li>
+ * <li> class name of {@link org.apache.carbondata.core.datamap.dev.DataMapFactory}
+ * implementation: Developer can implement new type of DataMap by extending
+ * {@link org.apache.carbondata.core.datamap.dev.DataMapFactory} </li>
* </ol>
*
* @since 1.4.0
*/
-@InterfaceAudience.Developer("DataMap")
-@InterfaceStability.Unstable
+@InterfaceAudience.Internal
public interface DataMapProvider {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
index e11e522..2a6a70a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
@@ -22,7 +22,7 @@ import org.apache.carbondata.common.exceptions.MetadataProcessException;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.core.datamap.DataMapRegistry;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.format.TableInfo;
@@ -38,7 +38,7 @@ public class IndexDataMapProvider implements DataMapProvider {
@Override
public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
SparkSession sparkSession) throws MalformedDataMapCommandException {
- IndexDataMapFactory dataMapFactory = createIndexDataMapFactory(dataMapSchema);
+ DataMapFactory dataMapFactory = createIndexDataMapFactory(dataMapSchema);
DataMapStoreManager.getInstance().registerDataMap(
mainTable.getAbsoluteTableIdentifier(), dataMapSchema, dataMapFactory);
originalTableInfo = PreAggregateUtil.updateMainTable(mainTable, dataMapSchema, sparkSession);
@@ -73,13 +73,13 @@ public class IndexDataMapProvider implements DataMapProvider {
throw new UnsupportedOperationException();
}
- private IndexDataMapFactory createIndexDataMapFactory(DataMapSchema dataMapSchema)
+ private DataMapFactory createIndexDataMapFactory(DataMapSchema dataMapSchema)
throws MalformedDataMapCommandException {
- IndexDataMapFactory dataMapFactory;
+ DataMapFactory dataMapFactory;
try {
// try to create DataMapProvider instance by taking providerName as class name
- Class<? extends IndexDataMapFactory> providerClass =
- (Class<? extends IndexDataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ Class<? extends DataMapFactory> providerClass =
+ (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
dataMapFactory = providerClass.newInstance();
} catch (ClassNotFoundException e) {
// try to create DataMapProvider instance by taking providerName as short name
@@ -91,14 +91,14 @@ public class IndexDataMapProvider implements DataMapProvider {
return dataMapFactory;
}
- private IndexDataMapFactory getDataMapFactoryByShortName(String providerName)
+ private DataMapFactory getDataMapFactoryByShortName(String providerName)
throws MalformedDataMapCommandException {
- IndexDataMapFactory dataMapFactory;
+ DataMapFactory dataMapFactory;
String className = DataMapRegistry.getDataMapClassName(providerName);
if (className != null) {
try {
- Class<? extends IndexDataMapFactory> datamapClass =
- (Class<? extends IndexDataMapFactory>) Class.forName(providerName);
+ Class<? extends DataMapFactory> datamapClass =
+ (Class<? extends DataMapFactory>) Class.forName(providerName);
dataMapFactory = datamapClass.newInstance();
} catch (ClassNotFoundException ex) {
throw new MalformedDataMapCommandException(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index be8d4d7..22a273b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -30,14 +30,14 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
-import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.processing.store.TablePage;
/**
- * It is for writing IndexDataMap for one table
+ * It is for writing DataMap for one table
*/
public class DataMapWriterListener {
@@ -45,26 +45,26 @@ public class DataMapWriterListener {
DataMapWriterListener.class.getCanonicalName());
// list indexed column name -> list of data map writer
- private Map<List<String>, List<AbstractDataMapWriter>> registry = new ConcurrentHashMap<>();
+ private Map<List<String>, List<DataMapWriter>> registry = new ConcurrentHashMap<>();
/**
* register all datamap writer for specified table and segment
*/
public void registerAllWriter(CarbonTable carbonTable, String segmentId,
String dataWritePath) {
- List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable);
- if (tableDataMaps != null) {
- for (TableDataMap tableDataMap : tableDataMaps) {
- IndexDataMapFactory factory = tableDataMap.getIndexDataMapFactory();
+ List<TableDataMap> tableIndices = DataMapStoreManager.getInstance().getAllDataMap(carbonTable);
+ if (tableIndices != null) {
+ for (TableDataMap tableDataMap : tableIndices) {
+ DataMapFactory factory = tableDataMap.getDataMapFactory();
register(factory, segmentId, dataWritePath);
}
}
}
/**
- * Register a AbstractDataMapWriter
+ * Register a DataMapWriter
*/
- private void register(IndexDataMapFactory factory, String segmentId, String dataWritePath) {
+ private void register(DataMapFactory factory, String segmentId, String dataWritePath) {
assert (factory != null);
assert (segmentId != null);
DataMapMeta meta = factory.getMeta();
@@ -73,9 +73,8 @@ public class DataMapWriterListener {
return;
}
List<String> columns = factory.getMeta().getIndexedColumns();
- List<AbstractDataMapWriter> writers = registry.get(columns);
- AbstractDataMapWriter writer = factory.createWriter(
- new Segment(segmentId, null), dataWritePath);
+ List<DataMapWriter> writers = registry.get(columns);
+ DataMapWriter writer = factory.createWriter(new Segment(segmentId, null), dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
@@ -83,36 +82,36 @@ public class DataMapWriterListener {
writers.add(writer);
registry.put(columns, writers);
}
- LOG.info("AbstractDataMapWriter " + writer + " added");
+ LOG.info("DataMapWriter " + writer + " added");
}
public void onBlockStart(String blockId, String blockPath) {
- for (List<AbstractDataMapWriter> writers : registry.values()) {
- for (AbstractDataMapWriter writer : writers) {
+ for (List<DataMapWriter> writers : registry.values()) {
+ for (DataMapWriter writer : writers) {
writer.onBlockStart(blockId);
}
}
}
public void onBlockEnd(String blockId) {
- for (List<AbstractDataMapWriter> writers : registry.values()) {
- for (AbstractDataMapWriter writer : writers) {
+ for (List<DataMapWriter> writers : registry.values()) {
+ for (DataMapWriter writer : writers) {
writer.onBlockEnd(blockId);
}
}
}
public void onBlockletStart(int blockletId) {
- for (List<AbstractDataMapWriter> writers : registry.values()) {
- for (AbstractDataMapWriter writer : writers) {
+ for (List<DataMapWriter> writers : registry.values()) {
+ for (DataMapWriter writer : writers) {
writer.onBlockletStart(blockletId);
}
}
}
public void onBlockletEnd(int blockletId) {
- for (List<AbstractDataMapWriter> writers : registry.values()) {
- for (AbstractDataMapWriter writer : writers) {
+ for (List<DataMapWriter> writers : registry.values()) {
+ for (DataMapWriter writer : writers) {
writer.onBlockletEnd(blockletId);
}
}
@@ -125,15 +124,15 @@ public class DataMapWriterListener {
* @param tablePage page data
*/
public void onPageAdded(int blockletId, int pageId, TablePage tablePage) {
- Set<Map.Entry<List<String>, List<AbstractDataMapWriter>>> entries = registry.entrySet();
- for (Map.Entry<List<String>, List<AbstractDataMapWriter>> entry : entries) {
+ Set<Map.Entry<List<String>, List<DataMapWriter>>> entries = registry.entrySet();
+ for (Map.Entry<List<String>, List<DataMapWriter>> entry : entries) {
List<String> indexedColumns = entry.getKey();
ColumnPage[] pages = new ColumnPage[indexedColumns.size()];
for (int i = 0; i < indexedColumns.size(); i++) {
pages[i] = tablePage.getColumnPage(indexedColumns.get(i));
}
- List<AbstractDataMapWriter> writers = entry.getValue();
- for (AbstractDataMapWriter writer : writers) {
+ List<DataMapWriter> writers = entry.getValue();
+ for (DataMapWriter writer : writers) {
writer.onPageAdded(blockletId, pageId, pages);
}
}
@@ -143,8 +142,8 @@ public class DataMapWriterListener {
* Finish all datamap writers
*/
public void finish() throws IOException {
- for (List<AbstractDataMapWriter> writers : registry.values()) {
- for (AbstractDataMapWriter writer : writers) {
+ for (List<DataMapWriter> writers : registry.values()) {
+ for (DataMapWriter writer : writers) {
writer.finish();
}
}
[30/54] [abbrv] carbondata git commit: [CARBONDATA-2172][Lucene] Add
text_columns property for Lucene DataMap
Posted by ra...@apache.org.
[CARBONDATA-2172][Lucene] Add text_columns property for Lucene DataMap
Add text_columns property for Lucene DataMap
This closes #2019
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d23f7fad
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d23f7fad
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d23f7fad
Branch: refs/heads/master
Commit: d23f7fad1f7db029d1dd0cc8e3db7a5b79463179
Parents: f9291cd
Author: QiangCai <qi...@qq.com>
Authored: Thu Mar 1 15:40:01 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../core/datamap/DataMapStoreManager.java | 16 ++--
.../core/datamap/dev/DataMapFactory.java | 4 +-
.../blockletindex/BlockletDataMapFactory.java | 2 +-
.../ThriftWrapperSchemaConverterImpl.java | 2 +-
.../schema/datamap/DataMapProvider.java | 4 +
.../schema/table/AggregationDataMapSchema.java | 2 +-
.../core/metadata/schema/table/CarbonTable.java | 16 +++-
.../metadata/schema/table/DataMapSchema.java | 44 ++++++++--
.../core/metadata/schema/table/TableInfo.java | 2 +-
.../core/metadata/schema/table/TableSchema.java | 16 ++--
datamap/lucene/pom.xml | 5 ++
.../lucene/LuceneDataMapFactoryBase.java | 89 ++++++++++++++------
.../lucene/LuceneFineGrainDataMapSuite.scala | 58 ++++++++++++-
.../preaggregate/TestPreAggCreateCommand.scala | 75 ++++-------------
.../preaggregate/TestPreAggregateLoad.scala | 7 +-
.../timeseries/TestTimeseriesDataLoad.scala | 6 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 4 +-
.../testsuite/datamap/DataMapWriterSuite.scala | 1 +
.../testsuite/datamap/FGDataMapTestCase.scala | 4 +-
.../iud/InsertOverwriteConcurrentTest.scala | 0
.../TestInsertAndOtherCommandConcurrent.scala | 38 ++++-----
.../carbondata/spark/load/ValidateUtil.scala | 0
.../carbondata/spark/util/DataLoadingUtil.scala | 0
.../carbondata/datamap/DataMapManager.java | 4 +-
.../datamap/IndexDataMapProvider.java | 6 +-
.../datamap/PreAggregateDataMapProvider.java | 2 +-
.../datamap/TimeseriesDataMapProvider.java | 4 +-
.../datamap/CarbonCreateDataMapCommand.scala | 21 +++--
.../datamap/CarbonDataMapShowCommand.scala | 2 +-
.../preaaggregate/PreAggregateListeners.scala | 9 +-
.../command/timeseries/TimeSeriesUtil.scala | 16 ++--
.../loading/model/CarbonLoadModelBuilder.java | 16 ++--
.../processing/loading/model/LoadOption.java | 23 ++++-
33 files changed, 311 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index ab339e8..a8d467f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -33,7 +33,6 @@ import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
@@ -89,8 +88,7 @@ public final class DataMapStoreManager {
List<TableDataMap> dataMaps = new ArrayList<>();
if (dataMapSchemaList != null) {
for (DataMapSchema dataMapSchema : dataMapSchemaList) {
- if (!dataMapSchema.getClassName().equalsIgnoreCase(
- DataMapProvider.PREAGGREGATE.toString())) {
+ if (dataMapSchema.isIndexDataMap()) {
dataMaps.add(getDataMap(carbonTable.getAbsoluteTableIdentifier(), dataMapSchema));
}
}
@@ -144,26 +142,28 @@ public final class DataMapStoreManager {
* Return a new datamap instance and registered in the store manager.
* The datamap is created using datamap name, datamap factory class and table identifier.
*/
- private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
+ // TODO: make it private
+ public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
DataMapSchema dataMapSchema) throws MalformedDataMapCommandException, IOException {
DataMapFactory dataMapFactory;
try {
// try to create datamap by reflection to test whether it is a valid DataMapFactory class
Class<? extends DataMapFactory> factoryClass =
- (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getProviderName());
dataMapFactory = factoryClass.newInstance();
} catch (ClassNotFoundException e) {
throw new MalformedDataMapCommandException(
- "DataMap '" + dataMapSchema.getClassName() + "' not found");
+ "DataMap '" + dataMapSchema.getProviderName() + "' not found");
} catch (Throwable e) {
throw new MetadataProcessException(
- "failed to create DataMap '" + dataMapSchema.getClassName() + "'", e);
+ "failed to create DataMap '" + dataMapSchema.getProviderName() + "'", e);
}
return registerDataMap(identifier, dataMapSchema, dataMapFactory);
}
public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier,
- DataMapSchema dataMapSchema, DataMapFactory dataMapFactory) throws IOException {
+ DataMapSchema dataMapSchema, DataMapFactory dataMapFactory)
+ throws IOException, MalformedDataMapCommandException {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index f2c376a..48038b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.datamap.DataMapMeta;
@@ -35,7 +36,8 @@ public interface DataMapFactory<T extends DataMap> {
/**
* Initialization of Datamap factory with the identifier and datamap name
*/
- void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) throws IOException;
+ void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema)
+ throws IOException, MalformedDataMapCommandException;
/**
* Return a new write for this datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index a383288..9674958 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -156,7 +156,7 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
Blocklet blocklet) throws IOException {
String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
- if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
+ if (identifier.getIndexFileName().equals(carbonIndexFileName)) {
DataMap dataMap = cache.get(identifier);
return ((BlockletDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index a15804e..8d19431 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -346,7 +346,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftChildSchema.setChildTableIdentifier(relationIdentifier);
}
thriftChildSchema.setProperties(wrapperChildSchema.getProperties());
- thriftChildSchema.setClassName(wrapperChildSchema.getClassName());
+ thriftChildSchema.setClassName(wrapperChildSchema.getProviderName());
thriftChildSchema.setDataMapName(wrapperChildSchema.getDataMapName());
if (wrapperChildSchema.getChildSchema() != null) {
thriftChildSchema.setChildTableSchema(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
index 0052428..39304d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/datamap/DataMapProvider.java
@@ -49,6 +49,10 @@ public enum DataMapProvider {
return className;
}
+ public String getShortName() {
+ return shortName;
+ }
+
private boolean isEqual(String dataMapClass) {
return (dataMapClass != null &&
(dataMapClass.equals(className) ||
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
index 2a16e1f..a43926c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/AggregationDataMapSchema.java
@@ -64,7 +64,7 @@ public class AggregationDataMapSchema extends DataMapSchema {
private Set aggExpToColumnMapping;
- public AggregationDataMapSchema(String dataMapName, String className) {
+ AggregationDataMapSchema(String dataMapName, String className) {
super(dataMapName, className);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index f81f1dd..f14672f 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -151,7 +151,7 @@ public class CarbonTable implements Serializable {
List<DataMapSchema> dataMapSchemas = new ArrayList<>();
for (DataMapSchema dataMapSchema : tableInfo.getDataMapSchemaList()) {
DataMapSchema newDataMapSchema = DataMapSchemaFactory.INSTANCE
- .getDataMapSchema(dataMapSchema.getDataMapName(), dataMapSchema.getClassName());
+ .getDataMapSchema(dataMapSchema.getDataMapName(), dataMapSchema.getProviderName());
newDataMapSchema.setChildSchema(dataMapSchema.getChildSchema());
newDataMapSchema.setProperties(dataMapSchema.getProperties());
newDataMapSchema.setRelationIdentifier(dataMapSchema.getRelationIdentifier());
@@ -641,6 +641,10 @@ public class CarbonTable implements Serializable {
return null != partitionInfo && partitionInfo.getPartitionType() == PartitionType.NATIVE_HIVE;
}
+ public PartitionInfo getPartitionInfo() {
+ return tablePartitionMap.get(getTableName());
+ }
+
/**
* @return absolute table identifier
*/
@@ -798,6 +802,16 @@ public class CarbonTable implements Serializable {
return hasDataMapSchema;
}
+ public DataMapSchema getDataMapSchema(String dataMapName) {
+ List<DataMapSchema> dataMaps = tableInfo.getDataMapSchemaList();
+ for (DataMapSchema dataMap : dataMaps) {
+ if (dataMap.getDataMapName().equalsIgnoreCase(dataMapName)) {
+ return dataMap;
+ }
+ }
+ return null;
+ }
+
public boolean isChildDataMap() {
return null != tableInfo.getParentRelationIdentifiers() &&
!tableInfo.getParentRelationIdentifiers().isEmpty();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
index ae49467..877fab7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -23,6 +23,8 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
+import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
+
/**
* Child schema class to maintain the child table details inside parent table
*/
@@ -32,10 +34,18 @@ public class DataMapSchema implements Serializable, Writable {
protected String dataMapName;
- // this name can be class name of the DataMapProvider implementation or short name of it
- private String className;
+ /**
+ * There are two kind of DataMaps:
+ * 1. Index DataMap: provider name is class name of implementation class of DataMapFactory
+ * 2. OLAP DataMap: provider name is one of the {@link DataMapProvider#shortName}
+ */
+ private String providerName;
+
+ /**
+ * identifier of the parent table
+ */
+ private RelationIdentifier relationIdentifier;
- protected RelationIdentifier relationIdentifier;
/**
* child table schema
*/
@@ -46,16 +56,19 @@ public class DataMapSchema implements Serializable, Writable {
*/
protected Map<String, String> properties;
+ /**
+ * WARN: This constructor should be used by deserialization only
+ */
public DataMapSchema() {
}
- public DataMapSchema(String dataMapName, String className) {
+ public DataMapSchema(String dataMapName, String providerName) {
this.dataMapName = dataMapName;
- this.className = className;
+ this.providerName = providerName;
}
- public String getClassName() {
- return className;
+ public String getProviderName() {
+ return providerName;
}
public TableSchema getChildSchema() {
@@ -86,9 +99,22 @@ public class DataMapSchema implements Serializable, Writable {
this.properties = properties;
}
+ /**
+ * Return true if this datamap is an Index DataMap
+ * @return
+ */
+ public boolean isIndexDataMap() {
+ if (providerName.equalsIgnoreCase(DataMapProvider.PREAGGREGATE.getShortName()) ||
+ providerName.equalsIgnoreCase(DataMapProvider.TIMESERIES.getShortName())) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
@Override public void write(DataOutput out) throws IOException {
out.writeUTF(dataMapName);
- out.writeUTF(className);
+ out.writeUTF(providerName);
boolean isRelationIdentifierExists = null != relationIdentifier;
out.writeBoolean(isRelationIdentifierExists);
if (isRelationIdentifierExists) {
@@ -112,7 +138,7 @@ public class DataMapSchema implements Serializable, Writable {
@Override public void readFields(DataInput in) throws IOException {
this.dataMapName = in.readUTF();
- this.className = in.readUTF();
+ this.providerName = in.readUTF();
boolean isRelationIdnentifierExists = in.readBoolean();
if (isRelationIdnentifierExists) {
this.relationIdentifier = new RelationIdentifier(null, null, null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 0d796c7..47e09d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -275,7 +275,7 @@ public class TableInfo implements Serializable, Writable {
DataMapSchema childSchema = new DataMapSchema();
childSchema.readFields(in);
DataMapSchema dataMapSchema = DataMapSchemaFactory.INSTANCE
- .getDataMapSchema(childSchema.getDataMapName(), childSchema.getClassName());
+ .getDataMapSchema(childSchema.getDataMapName(), childSchema.getProviderName());
dataMapSchema.setChildSchema(childSchema.getChildSchema());
dataMapSchema.setRelationIdentifier(childSchema.getRelationIdentifier());
dataMapSchema.setProperties(childSchema.getProperties());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 5d79abc..f008821 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -278,13 +278,15 @@ public class TableSchema implements Serializable, Writable {
RelationIdentifier relationIdentifier =
new RelationIdentifier(databaseName, tableName, tableId);
Map<String, String> properties = new HashMap<>();
- properties.put("CHILD_SELECT QUERY",
- CarbonUtil.encodeToString(
- queryString.trim().getBytes(
- // replace = to with & as hive metastore does not allow = inside. For base 64
- // only = is allowed as special character , so replace with &
- CarbonCommonConstants.DEFAULT_CHARSET)).replace("=","&"));
- properties.put("QUERYTYPE", queryType);
+ if (queryString != null) {
+ properties.put(
+ "CHILD_SELECT QUERY",
+ CarbonUtil.encodeToString(queryString.trim().getBytes(
+ // replace = to with & as hive metastore does not allow = inside. For base 64
+ // only = is allowed as special character , so replace with &
+ CarbonCommonConstants.DEFAULT_CHARSET)).replace("=", "&"));
+ properties.put("QUERYTYPE", queryType);
+ }
DataMapSchema dataMapSchema = new DataMapSchema(dataMapName, className);
dataMapSchema.setProperties(properties);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/datamap/lucene/pom.xml
----------------------------------------------------------------------
diff --git a/datamap/lucene/pom.xml b/datamap/lucene/pom.xml
index ee504c6..4019065 100644
--- a/datamap/lucene/pom.xml
+++ b/datamap/lucene/pom.xml
@@ -26,6 +26,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.3.2</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 88e70d5..ab4e9ee 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.DataMapDistributable;
@@ -33,14 +34,15 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.events.Event;
+import org.apache.commons.lang.StringUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -50,6 +52,8 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
@InterfaceAudience.Internal
abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFactory<T> {
+ static final String TEXT_COLUMNS = "text_columns";
+
/**
* Logger
*/
@@ -77,7 +81,7 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFac
@Override
public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema)
- throws IOException {
+ throws IOException, MalformedDataMapCommandException {
Objects.requireNonNull(identifier);
Objects.requireNonNull(dataMapSchema);
@@ -98,34 +102,17 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFac
throw new IOException(errorMessage);
}
- TableInfo tableInfo = carbonTable.getTableInfo();
- List<ColumnSchema> lstCoumnSchemas = tableInfo.getFactTable().getListOfColumns();
-
- // currently add all columns into lucene indexer
- // TODO:only add index columns
- List<String> indexedColumns = new ArrayList<String>();
- for (ColumnSchema columnSchema : lstCoumnSchemas) {
- if (!columnSchema.isInvisible()) {
- indexedColumns.add(columnSchema.getColumnName());
- }
- }
-
- // get indexed columns
- // Map<String, String> properties = dataMapSchema.getProperties();
- // String columns = properties.get("text_column");
- // if (columns != null) {
- // String[] columnArray = columns.split(CarbonCommonConstants.COMMA, -1);
- // Collections.addAll(indexedColumns, columnArray);
- // }
+ // validate DataMapSchema and get index columns
+ List<String> indexedColumns = validateAndGetIndexedColumns(dataMapSchema, carbonTable);
// add optimizedOperations
List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>();
- // optimizedOperations.add(ExpressionType.EQUALS);
- // optimizedOperations.add(ExpressionType.GREATERTHAN);
- // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO);
- // optimizedOperations.add(ExpressionType.LESSTHAN);
- // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO);
- // optimizedOperations.add(ExpressionType.NOT);
+ // optimizedOperations.add(ExpressionType.EQUALS);
+ // optimizedOperations.add(ExpressionType.GREATERTHAN);
+ // optimizedOperations.add(ExpressionType.GREATERTHAN_EQUALTO);
+ // optimizedOperations.add(ExpressionType.LESSTHAN);
+ // optimizedOperations.add(ExpressionType.LESSTHAN_EQUALTO);
+ // optimizedOperations.add(ExpressionType.NOT);
optimizedOperations.add(ExpressionType.TEXT_MATCH);
this.dataMapMeta = new DataMapMeta(indexedColumns, optimizedOperations);
@@ -135,6 +122,52 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> implements DataMapFac
}
/**
+ * validate Lucene DataMap
+ * 1. require TEXT_COLUMNS property
+ * 2. TEXT_COLUMNS can't contains illegal argument(empty, blank)
+ * 3. TEXT_COLUMNS can't contains duplicate same columns
+ * 4. TEXT_COLUMNS should be exists in table columns
+ * 5. TEXT_COLUMNS support only String DataType columns
+ */
+ private List<String> validateAndGetIndexedColumns(DataMapSchema dataMapSchema,
+ CarbonTable carbonTable) throws MalformedDataMapCommandException {
+ String textColumnsStr = dataMapSchema.getProperties().get(TEXT_COLUMNS);
+ if (textColumnsStr == null || StringUtils.isBlank(textColumnsStr)) {
+ throw new MalformedDataMapCommandException(
+ "Lucene DataMap require proper TEXT_COLUMNS property.");
+ }
+ String[] textColumns = textColumnsStr.split(",", -1);
+ for (int i = 0; i < textColumns.length; i++) {
+ textColumns[i] = textColumns[i].trim().toLowerCase();
+ }
+ for (int i = 0; i < textColumns.length; i++) {
+ if (textColumns[i].isEmpty()) {
+ throw new MalformedDataMapCommandException("TEXT_COLUMNS contains illegal argument.");
+ }
+ for (int j = i + 1; j < textColumns.length; j++) {
+ if (textColumns[i].equals(textColumns[j])) {
+ throw new MalformedDataMapCommandException(
+ "TEXT_COLUMNS has duplicate columns :" + textColumns[i]);
+ }
+ }
+ }
+ List<String> textColumnList = new ArrayList<String>(textColumns.length);
+ for (int i = 0; i < textColumns.length; i++) {
+ CarbonColumn column = carbonTable.getColumnByName(carbonTable.getTableName(), textColumns[i]);
+ if (null == column) {
+ throw new MalformedDataMapCommandException("TEXT_COLUMNS: " + textColumns[i]
+ + " does not exist in table. Please check create DataMap statement.");
+ } else if (column.getDataType() != DataTypes.STRING) {
+ throw new MalformedDataMapCommandException(
+ "TEXT_COLUMNS only supports String column. " + "Unsupported column: " + textColumns[i]
+ + ", DataType: " + column.getDataType());
+ }
+ textColumnList.add(column.getColName());
+ }
+ return textColumnList;
+ }
+
+ /**
* Return a new write for this datamap
*/
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 5e28e8a..bfcfa67 100644
--- a/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/datamap/lucene/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -24,7 +24,7 @@ import scala.util.Random
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
@@ -42,9 +42,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
| TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
- }
- test("test lucene fine grain data map") {
sql("DROP TABLE IF EXISTS datamap_test")
sql(
"""
@@ -52,11 +50,65 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
| STORED BY 'carbondata'
| TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
""".stripMargin)
+ }
+
+ test("validate TEXT_COLUMNS DataMap property") {
+ // require TEXT_COLUMNS
+ var exception = intercept[MalformedDataMapCommandException](sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ """.stripMargin))
+
+ assertResult("Lucene DataMap require proper TEXT_COLUMNS property.")(exception.getMessage)
+
+ // illegal argumnet.
+ exception = intercept[MalformedDataMapCommandException](sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ | DMProperties('text_COLUMNS'='name, ')
+ """.stripMargin))
+
+ assertResult("TEXT_COLUMNS contains illegal argument.")(exception.getMessage)
+
+ // not exists
+ exception = intercept[MalformedDataMapCommandException](sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ | DMProperties('text_COLUMNS'='city,school')
+ """.stripMargin))
+
+ assertResult("TEXT_COLUMNS: school does not exist in table. Please check create DataMap statement.")(exception.getMessage)
+
+ // duplicate columns
+ exception = intercept[MalformedDataMapCommandException](sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ | DMProperties('text_COLUMNS'='name,city,name')
+ """.stripMargin))
+ assertResult("TEXT_COLUMNS has duplicate columns :name")(exception.getMessage)
+
+ // only support String DataType
+ exception = intercept[MalformedDataMapCommandException](sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test
+ | USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ | DMProperties('text_COLUMNS'='city,id')
+ """.stripMargin))
+
+ assertResult("TEXT_COLUMNS only supports String column. Unsupported column: id, DataType: INT")(exception.getMessage)
+ }
+
+ test("test lucene fine grain data map") {
sql(
s"""
| CREATE DATAMAP dm ON TABLE datamap_test
| USING 'org.apache.carbondata.datamap.lucene.LuceneFineGrainDataMapFactory'
+ | DMProperties('TEXT_COLUMNS'='Name , cIty')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index f208c92..50b8bec 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -19,7 +19,7 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.{AnalysisException, CarbonDatasourceHadoopRelation, Row}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
@@ -248,9 +248,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
s"""create datamap preagg_sum on table tbl_1 using 'preaggregate' as select mac,avg(age) from tbl_1 group by mac"""
.stripMargin)
sql(
- "create datamap agg2 on table tbl_1 using 'preaggregate' DMPROPERTIES ('timeseries" +
- ".eventTime'='prodate', 'timeseries.hierarchy'='hour=1,day=1,month=1,year=1') as select prodate," +
- "mac from tbl_1 group by prodate,mac")
+ "create datamap agg2 on table tbl_1 using 'preaggregate' as select prodate, mac from tbl_1 group by prodate,mac")
checkExistence(sql("show tables"), false, "tbl_1_preagg_sum","tbl_1_agg2_day","tbl_1_agg2_hour","tbl_1_agg2_month","tbl_1_agg2_year")
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,CarbonCommonConstants.CARBON_SHOW_DATAMAPS_DEFAULT)
}
@@ -303,8 +301,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
| GROUP BY column3,column5,column2
""".stripMargin)
}
- assert(e.getMessage.contains(
- s"Unknown datamap provider/class abc"))
+ assert(e.getMessage.contains(s"DataMap 'abc' not found"))
sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
}
@@ -324,66 +321,26 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS tbl_1")
sql("create table if not exists tbl_1(imei string,age int,mac string ,prodate timestamp,update timestamp,gamepoint double,contrid double) stored by 'carbondata' ")
sql(
- "create datamap agg2 on table tbl_1 using 'preaggregate' DMPROPERTIES ('timeseries" +
- ".eventTime'='prodate', 'timeseries.hierarchy'='hour=1,day=1,month=1,year=1') as select prodate," +
- "mac from tbl_1 group by prodate,mac")
+ "create datamap agg2 on table tbl_1 using 'preaggregate' as select prodate, mac from tbl_1 group by prodate,mac")
checkExistence(sql("show tables"), false, "tbl_1_agg2_day","tbl_1_agg2_hour","tbl_1_agg2_month","tbl_1_agg2_year")
}
test("test pre agg create table 21: should support 'if not exists'") {
- try {
- sql(
- """
- | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT
- | column3,
- | sum(column3),
- | column5,
- | sum(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
-
- sql(
- """
- | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT
- | column3,
- | sum(column3),
- | column5,
- | sum(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
- assert(true)
- } catch {
- case _: Exception =>
- assert(false)
- }
+ sql(
+ """
+ | CREATE DATAMAP IF NOT EXISTS agg0 ON TABLE mainTable
+ | USING 'preaggregate'
+ | AS SELECT
+ | column3,
+ | sum(column3),
+ | column5,
+ | sum(column5)
+ | FROM maintable
+ | GROUP BY column3,column5,column2
+ """.stripMargin)
sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
}
- test("test pre agg create table 22: don't support 'create datamap if exists'") {
- val e: Exception = intercept[AnalysisException] {
- sql(
- """
- | CREATE DATAMAP IF EXISTS agg0 ON TABLE mainTable
- | USING 'preaggregate'
- | AS SELECT
- | column3,
- | sum(column3),
- | column5,
- | sum(column5)
- | FROM maintable
- | GROUP BY column3,column5,column2
- """.stripMargin)
- assert(true)
- }
- assert(e.getMessage.contains("identifier matching regex"))
- sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
- }
test("test show tables filterted with datamaps") {
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SHOW_DATAMAPS,"false")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
index da1ffb5..9ab6db8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
@@ -21,10 +21,13 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
import org.scalatest.{BeforeAndAfterAll, Ignore}
+
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
+
class TestPreAggregateLoad extends QueryTest with BeforeAndAfterAll {
val testData = s"$resourcesPath/sample.csv"
@@ -389,7 +392,7 @@ test("check load and select for avg double datatype") {
| group by id
""".stripMargin)
- val e: Exception = intercept[TableAlreadyExistsException] {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
sql(
s"""
| create datamap preagg_sum
@@ -399,7 +402,7 @@ test("check load and select for avg double datatype") {
| group by id
""".stripMargin)
}
- assert(e.getMessage.contains("already exists in database"))
+ assert(e.getMessage.contains("DataMap name 'preagg_sum' already exist"))
checkAnswer(sql(s"select * from maintable_preagg_sum"),
Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
sql("drop table if exists maintable")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
index b43b93b..c5f07a0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.util.SparkUtil4Test
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.util.CarbonProperties
@@ -270,7 +271,7 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
- val e: Exception = intercept[TableAlreadyExistsException] {
+ val e: Exception = intercept[MalformedDataMapCommandException] {
sql(
s"""
| CREATE DATAMAP agg0_second ON TABLE mainTable
@@ -282,7 +283,8 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
| GROUP BY mytime
""".stripMargin)
}
- assert(e.getMessage.contains("already exists in database"))
+ assert(e.getMessage.contains("DataMap name 'agg0_second' already exist"))
+ sql("DROP DATAMAP agg0_second ON TABLE mainTable")
}
test("create datamap with 'if not exists' after load data into mainTable and create datamap") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index 3cba3ff..b19c80d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -350,7 +350,9 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
// register datamap writer
- sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"create datamap cgdatamap on table datamap_test_cg " +
+ s"using '${classOf[CGDataMapFactory].getName}' " +
+ s"DMPROPERTIES('indexcolumns'='name')")
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
sql("select * from normal_test where name='n502670'"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index d5cae15..73e9fd9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datamap.dev.cgdatamap.{CoarseGrainDataMap, Coa
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index fae87a5..3bfc7b9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -175,7 +175,7 @@ class FGDataMap extends FineGrainDataMap {
override def prune(
filterExp: FilterResolverIntf,
segmentProperties: SegmentProperties,
- partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
+ partitions: java.util.List[PartitionSpec]): java.util.List[FineGrainBlocklet] = {
val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
val expression = filterExp.getFilterExpression
getEqualToExpression(expression, buffer)
@@ -189,7 +189,7 @@ class FGDataMap extends FineGrainDataMap {
}
private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
- value: Array[Byte]): Option[Blocklet] = {
+ value: Array[Byte]): Option[FineGrainBlocklet] = {
val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
val obj = new ObjectInputStream(outputStream)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index 3f0ca42..1f4b7fe 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -28,18 +28,16 @@ import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory}
+import org.apache.carbondata.core.datamap.dev.DataMapWriter
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{CoarseGrainDataMap, CoarseGrainDataMapFactory}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
-import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMap}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.{DataMapSchema, RelationIdentifier}
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
-import org.apache.carbondata.spark.testsuite.datamap.C2DataMapFactory
// This testsuite test insert and insert overwrite with other commands concurrently
class TestInsertAndOtherCommandConcurrent extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
@@ -50,11 +48,12 @@ class TestInsertAndOtherCommandConcurrent extends QueryTest with BeforeAndAfterA
dropTable()
buildTestData()
- // register hook to the table to sleep, thus the other command will be executed
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- AbsoluteTableIdentifier.from(storeLocation + "/orders", "default", "orders"),
- classOf[WaitingDataMap].getName,
- "test")
+ sql(
+ s"""
+ | create datamap test on table orders
+ | using '${classOf[WaitingDataMap].getName}'
+ | as select count(a) from hiveMetaStoreTable_1")
+ """.stripMargin)
}
private def buildTestData(): Unit = {
@@ -267,26 +266,22 @@ object Global {
var overwriteRunning = false
}
-class WaitingDataMap() extends AbstractCoarseGrainDataMapFactory {
+class WaitingDataMap() extends CoarseGrainDataMapFactory {
private var identifier: AbsoluteTableIdentifier = _
- override def init(identifier: AbsoluteTableIdentifier, dataMapName: String): Unit = {
- this.identifier = identifier
- }
-
override def fireEvent(event: Event): Unit = ???
override def clear(segmentId: Segment): Unit = {}
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainDataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): util.List[CoarseGrainDataMap] = ???
- override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainDataMap] = ???
+ override def getDataMaps(segment: Segment): util.List[CoarseGrainDataMap] = ???
- override def createWriter(segment: Segment, writeDirectoryPath: String): AbstractDataMapWriter = {
- new AbstractDataMapWriter(identifier, segment, writeDirectoryPath) {
+ override def createWriter(segment: Segment, writeDirectoryPath: String): DataMapWriter = {
+ new DataMapWriter(identifier, segment, writeDirectoryPath) {
override def onPageAdded(blockletId: Int, pageId: Int, pages: Array[ColumnPage]): Unit = { }
override def onBlockletEnd(blockletId: Int): Unit = { }
@@ -312,4 +307,9 @@ class WaitingDataMap() extends AbstractCoarseGrainDataMapFactory {
override def getMeta: DataMapMeta = new DataMapMeta(List("o_country").asJava, Seq(ExpressionType.EQUALS).asJava)
override def toDistributable(segmentId: Segment): util.List[DataMapDistributable] = ???
+
+ override def init(identifier: AbsoluteTableIdentifier,
+ dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/ValidateUtil.scala
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
index 2b3a306..b23d676 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/DataMapManager.java
@@ -40,9 +40,9 @@ public class DataMapManager {
*/
public DataMapProvider getDataMapProvider(DataMapSchema dataMapSchema) {
DataMapProvider provider;
- if (dataMapSchema.getClassName().equalsIgnoreCase(PREAGGREGATE.toString())) {
+ if (dataMapSchema.getProviderName().equalsIgnoreCase(PREAGGREGATE.toString())) {
provider = new PreAggregateDataMapProvider();
- } else if (dataMapSchema.getClassName().equalsIgnoreCase(TIMESERIES.toString())) {
+ } else if (dataMapSchema.getProviderName().equalsIgnoreCase(TIMESERIES.toString())) {
provider = new TimeseriesDataMapProvider();
} else {
provider = new IndexDataMapProvider();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
index c7651bb..1f075de 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/IndexDataMapProvider.java
@@ -81,14 +81,14 @@ public class IndexDataMapProvider implements DataMapProvider {
try {
// try to create DataMapProvider instance by taking providerName as class name
Class<? extends DataMapFactory> providerClass =
- (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getProviderName());
dataMapFactory = providerClass.newInstance();
} catch (ClassNotFoundException e) {
// try to create DataMapProvider instance by taking providerName as short name
- dataMapFactory = getDataMapFactoryByShortName(dataMapSchema.getClassName());
+ dataMapFactory = getDataMapFactoryByShortName(dataMapSchema.getProviderName());
} catch (Throwable e) {
throw new MetadataProcessException(
- "failed to create DataMapProvider '" + dataMapSchema.getClassName() + "'", e);
+ "failed to create DataMapProvider '" + dataMapSchema.getProviderName() + "'", e);
}
return dataMapFactory;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
index dc53900..c33354e 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/PreAggregateDataMapProvider.java
@@ -37,7 +37,7 @@ public class PreAggregateDataMapProvider implements DataMapProvider {
SparkSession sparkSession) throws MalformedDataMapCommandException {
validateDmProperty(dataMapSchema);
helper = new PreAggregateTableHelper(
- mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getClassName(),
+ mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getProviderName(),
dataMapSchema.getProperties(), ctasSqlStatement, null, false);
helper.initMeta(sparkSession);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java b/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
index a66f26a..f1575cd 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/datamap/TimeseriesDataMapProvider.java
@@ -36,13 +36,13 @@ public class TimeseriesDataMapProvider extends PreAggregateDataMapProvider {
public void initMeta(CarbonTable mainTable, DataMapSchema dataMapSchema, String ctasSqlStatement,
SparkSession sparkSession) {
Map<String, String> dmProperties = dataMapSchema.getProperties();
- String dmProviderName = dataMapSchema.getClassName();
+ String dmProviderName = dataMapSchema.getProviderName();
TimeSeriesUtil.validateTimeSeriesGranularity(dmProperties, dmProviderName);
Tuple2<String, String> details =
TimeSeriesUtil.getTimeSeriesGranularityDetails(dmProperties, dmProviderName);
dmProperties.remove(details._1());
helper = new PreAggregateTableHelper(
- mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getClassName(),
+ mainTable, dataMapSchema.getDataMapName(), dataMapSchema.getProviderName(),
dmProperties, ctasSqlStatement, new Some(details._1()), false);
helper.initMeta(sparkSession);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 8532e9d..8c475d4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -53,9 +53,17 @@ case class CarbonCreateDataMapCommand(
throw new MalformedCarbonCommandException("Streaming table does not support creating datamap")
}
- validateDataMapName(mainTable)
+ if (mainTable.getDataMapSchema(dataMapName) != null) {
+ if (!ifNotExistsSet) {
+ throw new MalformedDataMapCommandException(s"DataMap name '$dataMapName' already exist")
+ } else {
+ return Seq.empty
+ }
+ }
+
dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
- dataMapSchema.setProperties(new java.util.HashMap[String, String](dmProperties.asJava))
+ dataMapSchema.setProperties(new java.util.HashMap[String, String](
+ dmProperties.map(x => (x._1.trim, x._2.trim)).asJava))
dataMapProvider = DataMapManager.get().getDataMapProvider(dataMapSchema)
dataMapProvider.initMeta(mainTable, dataMapSchema, queryString.orNull, sparkSession)
@@ -64,15 +72,6 @@ case class CarbonCreateDataMapCommand(
Seq.empty
}
- private def validateDataMapName(carbonTable: CarbonTable): Unit = {
- val existingDataMaps = carbonTable.getTableInfo.getDataMapSchemaList
- existingDataMaps.asScala.foreach { dataMapSchema =>
- if (dataMapSchema.getDataMapName.equalsIgnoreCase(dataMapName)) {
- throw new MalformedDataMapCommandException(s"DataMap name '$dataMapName' already exist")
- }
- }
- }
-
override def processData(sparkSession: SparkSession): Seq[Row] = {
if (dataMapProvider != null) {
dataMapProvider.initData(mainTable, sparkSession)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
index fdd17d9..443ff70 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapShowCommand.scala
@@ -51,7 +51,7 @@ case class CarbonDataMapShowCommand(
if (relationIdentifier != null) {
table = relationIdentifier.getDatabaseName + "." + relationIdentifier.getTableName
}
- Row(s.getDataMapName, s.getClassName, table)
+ Row(s.getDataMapName, s.getProviderName, table)
}
} else {
Seq.empty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
index 11f451b..a516d1b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateListeners.scala
@@ -271,21 +271,22 @@ object CommitPreAggregateListener extends OperationEventListener {
operationContext.getProperty("isCompaction")).getOrElse("false").toString.toBoolean
val dataMapSchemas =
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getTableInfo.getDataMapSchemaList
+ .asScala.filter(_.getChildSchema != null)
// extract all child LoadCommands
val childLoadCommands = if (!isCompactionFlow) {
// If not compaction flow then the key for load commands will be tableName
- dataMapSchemas.asScala.map { dataMapSchema =>
+ dataMapSchemas.map { dataMapSchema =>
operationContext.getProperty(dataMapSchema.getChildSchema.getTableName)
.asInstanceOf[CarbonLoadDataCommand]
}
} else {
// If not compaction flow then the key for load commands will be tableName_Compaction
- dataMapSchemas.asScala.map { dataMapSchema =>
+ dataMapSchemas.map { dataMapSchema =>
operationContext.getProperty(dataMapSchema.getChildSchema.getTableName + "_Compaction")
.asInstanceOf[CarbonLoadDataCommand]
}
}
- if (dataMapSchemas.size() > 0) {
+ if (dataMapSchemas.nonEmpty) {
val uuid = operationContext.getProperty("uuid").toString
// keep committing until one fails
val renamedDataMaps = childLoadCommands.takeWhile { childLoadCommand =>
@@ -299,7 +300,7 @@ object CommitPreAggregateListener extends OperationEventListener {
renameDataMapTableStatusFiles(oldTableSchemaPath, newTableSchemaPath, uuid)
}
// if true then the commit for one of the child tables has failed
- val commitFailed = renamedDataMaps.lengthCompare(dataMapSchemas.size()) != 0
+ val commitFailed = renamedDataMaps.lengthCompare(dataMapSchemas.length) != 0
if (commitFailed) {
LOGGER.warn("Reverting table status file to original state")
renamedDataMaps.foreach {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index 6fd0820..5c23805 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.execution.command.timeseries
+import scala.collection.mutable
+
import org.apache.spark.sql.execution.command.{DataMapField, Field}
import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
@@ -172,14 +174,14 @@ object TimeSeriesUtil {
* @param fieldMapping fields from select plan
* @param timeSeriesColumn timeseries column name
*/
- def validateEventTimeColumnExitsInSelect(fieldMapping: scala.collection.mutable
- .LinkedHashMap[Field, DataMapField],
+ def validateEventTimeColumnExitsInSelect(
+ fieldMapping: mutable.LinkedHashMap[Field, DataMapField],
timeSeriesColumn: String) : Any = {
- val isTimeSeriesColumnExits = fieldMapping
- .exists(obj => obj._2.columnTableRelationList.isDefined &&
- obj._2.columnTableRelationList.get(0).parentColumnName
- .equalsIgnoreCase(timeSeriesColumn) &&
- obj._2.aggregateFunction.isEmpty)
+ val isTimeSeriesColumnExits = fieldMapping.exists { case (_, f) =>
+ f.columnTableRelationList.isDefined &&
+ f.columnTableRelationList.get.head.parentColumnName.equalsIgnoreCase(timeSeriesColumn) &&
+ f.aggregateFunction.isEmpty
+ }
if(!isTimeSeriesColumnExits) {
throw new MalformedCarbonCommandException(s"Time series column ${ timeSeriesColumn } does " +
s"not exists in select")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index 29dfa40..7871643 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -148,11 +148,12 @@ public class CarbonLoadModelBuilder {
if (Boolean.parseBoolean(bad_records_logger_enable) ||
LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
- bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path);
if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
throw new InvalidLoadOptionException("Invalid bad records location.");
}
+ bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path);
}
+
carbonLoadModel.setBadRecordsLocation(bad_record_path);
validateGlobalSortPartitions(global_sort_partitions);
@@ -226,10 +227,8 @@ public class CarbonLoadModelBuilder {
delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
throw new InvalidLoadOptionException("Field Delimiter and Complex types delimiter are same");
} else {
- carbonLoadModel.setComplexDelimiterLevel1(
- CarbonUtil.delimiterConverter(complex_delimeter_level1));
- carbonLoadModel.setComplexDelimiterLevel2(
- CarbonUtil.delimiterConverter(complex_delimeter_level2));
+ carbonLoadModel.setComplexDelimiterLevel1(complex_delimeter_level1);
+ carbonLoadModel.setComplexDelimiterLevel2(complex_delimeter_level2);
}
// set local dictionary path, and dictionary file extension
carbonLoadModel.setAllDictPath(all_dictionary_path);
@@ -320,10 +319,13 @@ public class CarbonLoadModelBuilder {
private void validateSortScope(String sortScope) throws InvalidLoadOptionException {
if (sortScope != null) {
- // Don't support use global sort on partitioned table.
+ // We support global sort for Hive standard partition, but don't support
+ // global sort for other partition type.
if (table.getPartitionInfo(table.getTableName()) != null &&
+ !table.isHivePartitionTable() &&
sortScope.equalsIgnoreCase(SortScopeOptions.SortScope.GLOBAL_SORT.toString())) {
- throw new InvalidLoadOptionException("Don't support use global sort on partitioned table.");
+ throw new InvalidLoadOptionException("Don't support use global sort on "
+ + table.getPartitionInfo().getPartitionType() + " partition table.");
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d23f7fad/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index bac1a94..608d147 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.processing.loading.model;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -112,11 +113,11 @@ public class LoadOption {
optionsFinal.put(
"complex_delimiter_level_1",
- Maps.getOrDefault(options,"complex_delimiter_level_1", "\\$"));
+ Maps.getOrDefault(options,"complex_delimiter_level_1", "$"));
optionsFinal.put(
"complex_delimiter_level_2",
- Maps.getOrDefault(options, "complex_delimiter_level_2", "\\:"));
+ Maps.getOrDefault(options, "complex_delimiter_level_2", ":"));
optionsFinal.put(
"dateformat",
@@ -259,7 +260,23 @@ public class LoadOption {
+ "the same. Input file : " + CarbonUtil.removeAKSK(csvFile));
}
}
- return csvColumns;
+
+ // In case of static partition columns just change the name of header if already exists as
+ // we should not take the column from csv file and add them as new columns at the end.
+ if (staticPartitionCols.size() > 0) {
+ List<String> updatedColumns = new ArrayList<>();
+ for (int i = 0; i < csvColumns.length; i++) {
+ if (staticPartitionCols.contains(csvColumns[i])) {
+ updatedColumns.add(csvColumns[i] + "1");
+ } else {
+ updatedColumns.add(csvColumns[i]);
+ }
+ }
+ updatedColumns.addAll(staticPartitionCols);
+ return updatedColumns.toArray(new String[updatedColumns.size()]);
+ } else {
+ return csvColumns;
+ }
}
}
[23/54] [abbrv] carbondata git commit: [CARBONDATA-2018][DataLoad]
Optimization in reading/writing for sort temp row
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index 6f05088..cbcbbae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -19,8 +19,9 @@ package org.apache.carbondata.processing.loading.sort.unsafe.holder;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
public class UnsafeInmemoryHolder implements SortTempChunkHolder {
@@ -33,21 +34,18 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
private UnsafeCarbonRowPage rowPage;
- private Object[] currentRow;
+ private IntermediateSortTempRow currentRow;
private long address;
- private NewRowComparator comparator;
+ private IntermediateSortTempRowComparator comparator;
- private int columnSize;
-
- public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize,
- int numberOfSortColumns) {
+ public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(rowPage.getNoDictionarySortColumnMapping());
- this.columnSize = columnSize;
+ this.comparator = new IntermediateSortTempRowComparator(
+ rowPage.getTableFieldStat().getIsSortColNoDictFlags());
}
public boolean hasNext() {
@@ -58,13 +56,12 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
}
public void readRow() {
- currentRow = new Object[columnSize];
address = rowPage.getBuffer().get(counter);
- rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset(), currentRow);
+ currentRow = rowPage.getRow(address + rowPage.getDataBlock().getBaseOffset());
counter++;
}
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return currentRow;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 11b3d43..527452a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -31,15 +31,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
-import org.apache.carbondata.processing.sort.sortdata.NewRowComparator;
+import org.apache.carbondata.processing.sort.sortdata.IntermediateSortTempRowComparator;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@@ -63,21 +62,15 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
* entry count
*/
private int entryCount;
-
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -100,27 +93,21 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
private int numberOfObjectRead;
- private int nullSetWordsLength;
-
- private Comparator<Object[]> comparator;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator<IntermediateSortTempRow> comparator;
/**
* Constructor to initialize
*/
public UnsafeSortTempFileChunkHolder(File tempFile, SortParameters parameters) {
// set temp file
this.tempFile = tempFile;
- this.dimCnt = parameters.getDimColCount();
- this.complexCnt = parameters.getComplexDimColCount();
- this.measureCnt = parameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = parameters.getMeasureDataType();
this.readBufferSize = parameters.getBufferSize();
this.compressorName = parameters.getSortTempCompressorName();
-
+ this.tableFieldStat = new TableFieldStat(parameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
this.executorService = Executors.newFixedThreadPool(1);
- this.nullSetWordsLength = ((parameters.getMeasureColCount() - 1) >> 6) + 1;
- comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
+ comparator = new IntermediateSortTempRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
@@ -169,11 +156,17 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @throws CarbonSortKeyAndGroupByException problem while reading
*/
+ @Override
public void readRow() throws CarbonSortKeyAndGroupByException {
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+ this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ this.numberOfObjectRead++;
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException("Problems while reading row", e);
+ }
}
}
@@ -207,63 +200,22 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
}
/**
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * get a batch of row, this interface is used in reading compressed sort temp files
+ *
+ * @param expected expected number in a batch
+ * @return a batch of row
+ * @throws IOException if error occurs while reading from stream
*/
- private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
- Object[] row = new Object[dimCnt + measureCnt];
- try {
- int dimCount = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- short aShort = stream.readShort();
- byte[] col = new byte[aShort];
- stream.readFully(col);
- row[dimCount] = col;
- } else {
- int anInt = stream.readInt();
- row[dimCount] = anInt;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimCnt; dimCount++) {
- short aShort = stream.readShort();
- byte[] col = new byte[aShort];
- stream.readFully(col);
- row[dimCount] = col;
- }
-
- long[] words = new long[nullSetWordsLength];
- for (int i = 0; i < words.length; i++) {
- words[i] = stream.readLong();
- }
-
- for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
- if (UnsafeCarbonRowPage.isSet(words, mesCount)) {
- DataType dataType = measureDataTypes[mesCount];
- if (dataType == DataTypes.SHORT) {
- row[dimCount + mesCount] = stream.readShort();
- } else if (dataType == DataTypes.INT) {
- row[dimCount + mesCount] = stream.readInt();
- } else if (dataType == DataTypes.LONG) {
- row[dimCount + mesCount] = stream.readLong();
- } else if (dataType == DataTypes.DOUBLE) {
- row[dimCount + mesCount] = stream.readDouble();
- } else if (DataTypes.isDecimal(dataType)) {
- short aShort = stream.readShort();
- byte[] bigDecimalInBytes = new byte[aShort];
- stream.readFully(bigDecimalInBytes);
- row[dimCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- }
- }
- return row;
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException(e);
+ private IntermediateSortTempRow[] readBatchedRowFromStream(int expected)
+ throws IOException {
+ IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
+ for (int i = 0; i < expected; i++) {
+ IntermediateSortTempRow holder
+ = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ holders[i] = holder;
}
+ this.numberOfObjectRead += expected;
+ return holders;
}
/**
@@ -271,7 +223,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
*
* @return row
*/
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return this.returnRow;
}
@@ -326,9 +278,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
@Override public int hashCode() {
int hash = 0;
- hash += 31 * measureCnt;
- hash += 31 * dimCnt;
- hash += 31 * complexCnt;
+ hash += tableFieldStat.hashCode();
hash += tempFile.hashCode();
return hash;
}
@@ -368,16 +318,12 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * @param numberOfRecords number of records to be read
+ * @return batch of intermediate sort temp row
+ * @throws IOException if error occurs reading records from file
*/
- private Object[][] prefetchRecordsFromFile(int numberOfRecords)
- throws CarbonSortKeyAndGroupByException {
- Object[][] records = new Object[numberOfRecords][];
- for (int i = 0; i < numberOfRecords; i++) {
- records[i] = getRowFromStream();
- }
- return records;
+ private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
+ throws IOException {
+ return readBatchedRowFromStream(numberOfRecords);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index 4bbf61b..22673ff 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -21,25 +21,21 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
import java.util.AbstractQueue;
-import java.util.Arrays;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeSortTempFileChunkHolder;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
@@ -69,22 +65,13 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
private int totalNumberOfRecords;
private SortParameters mergerParameters;
-
+ private TableFieldStat tableFieldStat;
private File[] intermediateFiles;
-
private File outPutFile;
- private int dimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
-
- private long[] nullSetWords;
-
- private ByteBuffer rowData;
+ private SortStepRowHandler sortStepRowHandler;
private Throwable throwable;
@@ -97,16 +84,10 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
- this.dimCnt = mergerParameters.getDimColCount();
- this.complexCnt = mergerParameters.getComplexDimColCount();
- this.measureCnt = mergerParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
- this.nullSetWords = new long[((measureCnt - 1) >> 6) + 1];
- // Take size of 2 MB for each row. I think it is high enough to use
- rowData = ByteBuffer.allocate(2 * 1024 * 1024);
+ this.tableFieldStat = new TableFieldStat(mergerParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
}
@Override public Void call() throws Exception {
@@ -165,13 +146,14 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted record from file
+ * This method will be used to get sorted sort temp row from the sort temp files
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile()
+ throws CarbonSortKeyAndGroupByException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -235,7 +217,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
}
/**
@@ -250,12 +232,12 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] next() throws CarbonSortKeyAndGroupByException {
+ private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -272,82 +254,16 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws CarbonSortKeyAndGroupByException problem while writing
+ * @throws IOException problem while writing
*/
- private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException, IOException {
- int dimCount = 0;
- int size = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- rowData.putShort((short) col.length);
- size += 2;
- rowData.put(col);
- size += col.length;
- } else {
- rowData.putInt((int) row[dimCount]);
- size += 4;
- }
- }
-
- // write complex dimensions here.
- int dimensionSize = dimCnt + complexCnt;
- for (; dimCount < dimensionSize; dimCount++) {
- byte[] col = (byte[]) row[dimCount];
- rowData.putShort((short)col.length);
- size += 2;
- rowData.put(col);
- size += col.length;
- }
- Arrays.fill(nullSetWords, 0);
- int nullSetSize = nullSetWords.length * 8;
- int nullLoc = size;
- size += nullSetSize;
- for (int mesCount = 0; mesCount < measureCnt; mesCount++) {
- Object value = row[mesCount + dimensionSize];
- if (null != value) {
- DataType dataType = measureDataTypes[mesCount];
- if (dataType == DataTypes.SHORT) {
- rowData.putShort(size, (Short) value);
- size += 2;
- } else if (dataType == DataTypes.INT) {
- rowData.putInt(size, (Integer) value);
- size += 4;
- } else if (dataType == DataTypes.LONG) {
- rowData.putLong(size, (Long) value);
- size += 8;
- } else if (dataType == DataTypes.DOUBLE) {
- rowData.putDouble(size, (Double) value);
- size += 8;
- } else if (DataTypes.isDecimal(dataType)) {
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(((BigDecimal) value));
- rowData.putShort(size, (short) bigDecimalInBytes.length);
- size += 2;
- for (int i = 0; i < bigDecimalInBytes.length; i++) {
- rowData.put(size++, bigDecimalInBytes[i]);
- }
- }
- UnsafeCarbonRowPage.set(nullSetWords, mesCount);
- } else {
- UnsafeCarbonRowPage.unset(nullSetWords, mesCount);
- }
- }
- for (int i = 0; i < nullSetWords.length; i++) {
- rowData.putLong(nullLoc, nullSetWords[i]);
- nullLoc += 8;
- }
- byte[] rowBytes = new byte[size];
- rowData.position(0);
- rowData.get(rowBytes);
- stream.write(rowBytes);
- rowData.clear();
+ private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
}
private void finish() throws CarbonSortKeyAndGroupByException {
clear();
try {
CarbonUtil.deleteFiles(intermediateFiles);
- rowData.clear();
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate files");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index ce118d9..64f3c25 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -29,7 +29,8 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
-import org.apache.carbondata.processing.loading.sort.SortStepRowUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder;
import org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder;
@@ -55,7 +56,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
private AbstractQueue<SortTempChunkHolder> recordHolderHeapLocal;
private SortParameters parameters;
- private SortStepRowUtil sortStepRowUtil;
+ private SortStepRowHandler sortStepRowHandler;
/**
* tempFileLocation
*/
@@ -68,7 +69,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
public UnsafeSingleThreadFinalSortFilesMerger(SortParameters parameters,
String[] tempFileLocation) {
this.parameters = parameters;
- this.sortStepRowUtil = new SortStepRowUtil(parameters);
+ this.sortStepRowHandler = new SortStepRowHandler(parameters);
this.tempFileLocation = tempFileLocation;
this.tableName = parameters.getTableName();
}
@@ -108,9 +109,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
LOGGER.info("Started adding first record from each page");
for (final UnsafeCarbonRowPage rowPage : rowPages) {
- SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage,
- parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount(), parameters.getNumberOfSortColumns());
+ SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
// initialize
sortTempFileChunkHolder.readRow();
@@ -121,9 +120,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder =
- new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(),
- parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount());
+ new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn());
// initialize
sortTempFileChunkHolder.readRow();
@@ -142,7 +139,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeapLocal.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
} catch (Exception e) {
LOGGER.error(e);
throw new CarbonDataWriterException(e);
@@ -180,12 +177,14 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted row in 3-parted format.
+ * The row will feed the following writer process step.
*
* @return sorted row
*/
public Object[] next() {
- return sortStepRowUtil.convertRow(getSortedRecordFromFile());
+ IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
+ return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
}
/**
@@ -193,8 +192,8 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
*
* @return sorted record sorted record
*/
- private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index 324df2a..bce1b33 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -389,7 +389,6 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
noDictionarySortColumnMapping, 0, noDictionarySortColumnMapping.length);
}
sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
-
String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation,
CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
finalMerger =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
index 04efa1f..c06819c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java
@@ -21,7 +21,6 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.math.BigDecimal;
import java.util.AbstractQueue;
import java.util.PriorityQueue;
import java.util.concurrent.Callable;
@@ -29,11 +28,9 @@ import java.util.concurrent.Callable;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class IntermediateFileMerger implements Callable<Void> {
@@ -68,17 +65,12 @@ public class IntermediateFileMerger implements Callable<Void> {
private File[] intermediateFiles;
private File outPutFile;
- private int dimCnt;
- private int noDictDimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private DataType[] measureDataTypes;
private int writeBufferSize;
private String compressorName;
private Throwable throwable;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
/**
* IntermediateFileMerger Constructor
*/
@@ -88,14 +80,10 @@ public class IntermediateFileMerger implements Callable<Void> {
this.fileCounter = intermediateFiles.length;
this.intermediateFiles = intermediateFiles;
this.outPutFile = outPutFile;
- this.dimCnt = mergerParameters.getDimColCount();
- this.noDictDimCnt = mergerParameters.getNoDictionaryCount();
- this.complexCnt = mergerParameters.getComplexDimColCount();
- this.measureCnt = mergerParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = mergerParameters.getNoDictionaryDimnesionColumn();
- this.measureDataTypes = mergerParameters.getMeasureDataType();
this.writeBufferSize = mergerParameters.getBufferSize();
this.compressorName = mergerParameters.getSortTempCompressorName();
+ this.tableFieldStat = new TableFieldStat(mergerParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
}
@Override public Void call() throws Exception {
@@ -154,13 +142,14 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted record from file
+ * This method will be used to get the sorted sort temp row from sort temp file
*
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonSortKeyAndGroupByException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile()
+ throws CarbonSortKeyAndGroupByException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
@@ -227,7 +216,7 @@ public class IntermediateFileMerger implements Callable<Void> {
this.recordHolderHeap.add(sortTempFileChunkHolder);
}
- LOGGER.info("Heap Size" + this.recordHolderHeap.size());
+ LOGGER.info("Heap Size: " + this.recordHolderHeap.size());
}
/**
@@ -242,12 +231,12 @@ public class IntermediateFileMerger implements Callable<Void> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] next() throws CarbonSortKeyAndGroupByException {
+ private IntermediateSortTempRow next() throws CarbonSortKeyAndGroupByException {
return getSortedRecordFromFile();
}
@@ -264,62 +253,10 @@ public class IntermediateFileMerger implements Callable<Void> {
/**
* Below method will be used to write data to file
*
- * @throws CarbonSortKeyAndGroupByException problem while writing
+ * @throws IOException problem while writing
*/
- private void writeDataToFile(Object[] row) throws CarbonSortKeyAndGroupByException {
- try {
- int[] mdkArray = (int[]) row[0];
- byte[][] nonDictArray = (byte[][]) row[1];
- int mdkIndex = 0;
- int nonDictKeyIndex = 0;
- // write dictionary and non dictionary dimensions here.
- for (boolean nodictinary : isNoDictionaryDimensionColumn) {
- if (nodictinary) {
- byte[] col = nonDictArray[nonDictKeyIndex++];
- stream.writeShort(col.length);
- stream.write(col);
- } else {
- stream.writeInt(mdkArray[mdkIndex++]);
- }
- }
- // write complex
- for (; nonDictKeyIndex < noDictDimCnt + complexCnt; nonDictKeyIndex++) {
- byte[] col = nonDictArray[nonDictKeyIndex++];
- stream.writeShort(col.length);
- stream.write(col);
- }
- // write measure
- int fieldIndex = 0;
- for (int counter = 0; counter < measureCnt; counter++) {
- if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) {
- stream.write((byte) 1);
- DataType dataType = measureDataTypes[counter];
- if (dataType == DataTypes.BOOLEAN) {
- stream.writeBoolean((boolean)NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.SHORT) {
- stream.writeShort((short) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.INT) {
- stream.writeInt((int) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.LONG) {
- stream.writeLong((long) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (dataType == DataTypes.DOUBLE) {
- stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row));
- } else if (DataTypes.isDecimal(dataType)) {
- byte[] bigDecimalInBytes = DataTypeUtil
- .bigDecimalToByte((BigDecimal) NonDictionaryUtil.getMeasure(fieldIndex, row));
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- } else {
- stream.write((byte) 0);
- }
- fieldIndex++;
- }
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
- }
+ private void writeDataToFile(IntermediateSortTempRow row) throws IOException {
+ sortStepRowHandler.writeIntermediateSortTempRowToOutputStream(row, stream);
}
private void finish() throws CarbonSortKeyAndGroupByException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
new file mode 100644
index 0000000..9b6d1e8
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateSortTempRowComparator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+
+/**
+ * This class is used as comparator for comparing intermediate sort temp row
+ */
+public class IntermediateSortTempRowComparator implements Comparator<IntermediateSortTempRow> {
+ /**
+ * isSortColumnNoDictionary whether the sort column is not dictionary or not
+ */
+ private boolean[] isSortColumnNoDictionary;
+
+ /**
+ * @param isSortColumnNoDictionary isSortColumnNoDictionary
+ */
+ public IntermediateSortTempRowComparator(boolean[] isSortColumnNoDictionary) {
+ this.isSortColumnNoDictionary = isSortColumnNoDictionary;
+ }
+
+ /**
+ * Below method will be used to compare two sort temp row
+ */
+ public int compare(IntermediateSortTempRow rowA, IntermediateSortTempRow rowB) {
+ int diff = 0;
+ int dictIndex = 0;
+ int nonDictIndex = 0;
+
+ for (boolean isNoDictionary : isSortColumnNoDictionary) {
+
+ if (isNoDictionary) {
+ byte[] byteArr1 = rowA.getNoDictSortDims()[nonDictIndex];
+ byte[] byteArr2 = rowB.getNoDictSortDims()[nonDictIndex];
+ nonDictIndex++;
+
+ int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
+ if (difference != 0) {
+ return difference;
+ }
+ } else {
+ int dimFieldA = rowA.getDictSortDims()[dictIndex];
+ int dimFieldB = rowB.getDictSortDims()[dictIndex];
+ dictIndex++;
+
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ }
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
index d2579d2..3f94533 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparator.java
@@ -40,14 +40,11 @@ public class NewRowComparator implements Comparator<Object[]> {
*/
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
-
int index = 0;
for (boolean isNoDictionary : noDictionarySortColumnMaping) {
-
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[index];
-
byte[] byteArr2 = (byte[]) rowB[index];
int difference = UnsafeComparer.INSTANCE.compareTo(byteArr1, byteArr2);
@@ -57,6 +54,7 @@ public class NewRowComparator implements Comparator<Object[]> {
} else {
int dimFieldA = (int) rowA[index];
int dimFieldB = (int) rowB[index];
+
diff = dimFieldA - dimFieldB;
if (diff != 0) {
return diff;
@@ -65,7 +63,6 @@ public class NewRowComparator implements Comparator<Object[]> {
index++;
}
-
return diff;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
index e01b587..7538c92 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/NewRowComparatorForNormalDims.java
@@ -29,7 +29,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
private int numberOfSortColumns;
/**
- * RowComparatorForNormalDims Constructor
+ * NewRowComparatorForNormalDims Constructor
*
* @param numberOfSortColumns
*/
@@ -46,7 +46,6 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
int diff = 0;
for (int i = 0; i < numberOfSortColumns; i++) {
-
int dimFieldA = (int)rowA[i];
int dimFieldB = (int)rowB[i];
diff = dimFieldA - dimFieldB;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
deleted file mode 100644
index 0ae0b93..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.nio.ByteBuffer;
-import java.util.Comparator;
-
-import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-
-public class RowComparator implements Comparator<Object[]> {
- /**
- * noDictionaryCount represent number of no dictionary cols
- */
- private int noDictionaryCount;
-
- /**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
- */
- private boolean[] noDictionarySortColumnMaping;
-
- /**
- * @param noDictionarySortColumnMaping
- * @param noDictionaryCount
- */
- public RowComparator(boolean[] noDictionarySortColumnMaping, int noDictionaryCount) {
- this.noDictionaryCount = noDictionaryCount;
- this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
- }
-
- /**
- * Below method will be used to compare two mdkey
- */
- public int compare(Object[] rowA, Object[] rowB) {
- int diff = 0;
-
- int normalIndex = 0;
- int noDictionaryindex = 0;
-
- for (boolean isNoDictionary : noDictionarySortColumnMaping) {
-
- if (isNoDictionary) {
- byte[] byteArr1 = (byte[]) rowA[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-
- ByteBuffer buff1 = ByteBuffer.wrap(byteArr1);
-
- // extract a high card dims from complete byte[].
- NonDictionaryUtil
- .extractSingleHighCardDims(byteArr1, noDictionaryindex, noDictionaryCount, buff1);
-
- byte[] byteArr2 = (byte[]) rowB[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
-
- ByteBuffer buff2 = ByteBuffer.wrap(byteArr2);
-
- // extract a high card dims from complete byte[].
- NonDictionaryUtil
- .extractSingleHighCardDims(byteArr2, noDictionaryindex, noDictionaryCount, buff2);
-
- int difference = UnsafeComparer.INSTANCE.compareTo(buff1, buff2);
- if (difference != 0) {
- return difference;
- }
- noDictionaryindex++;
- } else {
- int dimFieldA = NonDictionaryUtil.getDimension(normalIndex, rowA);
- int dimFieldB = NonDictionaryUtil.getDimension(normalIndex, rowB);
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- normalIndex++;
- }
-
- }
-
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
deleted file mode 100644
index 0883ae1..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.util.Comparator;
-
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-
-/**
- * This class is used as comparator for comparing dims which are non high cardinality dims.
- * Here the dims will be in form of int[] (surrogates) so directly comparing the integers.
- */
-public class RowComparatorForNormalDims implements Comparator<Object[]> {
- /**
- * dimension count
- */
- private int numberOfSortColumns;
-
- /**
- * RowComparatorForNormalDims Constructor
- *
- * @param numberOfSortColumns
- */
- public RowComparatorForNormalDims(int numberOfSortColumns) {
- this.numberOfSortColumns = numberOfSortColumns;
- }
-
- /**
- * Below method will be used to compare two surrogate keys
- *
- * @see Comparator#compare(Object, Object)
- */
- public int compare(Object[] rowA, Object[] rowB) {
- int diff = 0;
-
- for (int i = 0; i < numberOfSortColumns; i++) {
-
- int dimFieldA = NonDictionaryUtil.getDimension(i, rowA);
- int dimFieldB = NonDictionaryUtil.getDimension(i, rowB);
-
- diff = dimFieldA - dimFieldB;
- if (diff != 0) {
- return diff;
- }
- }
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
index 88695b9..a4ac0ea 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
@@ -37,6 +37,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -71,12 +73,12 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* tableName
*/
private String tableName;
-
+ private SortParameters sortParameters;
+ private SortStepRowHandler sortStepRowHandler;
/**
* tempFileLocation
*/
private String[] tempFileLocation;
- private SortParameters sortParameters;
private int maxThreadForSorting;
@@ -89,6 +91,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.sortParameters = sortParameters;
+ this.sortStepRowHandler = new SortStepRowHandler(sortParameters);
try {
maxThreadForSorting = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD,
@@ -107,8 +110,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
public void startFinalMerge() throws CarbonDataWriterException {
List<File> filesToMerge = getFilesToMergeSort();
- if (filesToMerge.size() == 0)
- {
+ if (filesToMerge.size() == 0) {
LOGGER.info("No file to merge in final merge stage");
return;
}
@@ -125,11 +127,9 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// get all the merged files
List<File> files = new ArrayList<File>(tempFileLocation.length);
- for (String tempLoc : tempFileLocation)
- {
+ for (String tempLoc : tempFileLocation) {
File[] subFiles = new File(tempLoc).listFiles(fileFilter);
- if (null != subFiles && subFiles.length > 0)
- {
+ if (null != subFiles && subFiles.length > 0) {
files.addAll(Arrays.asList(subFiles));
}
}
@@ -226,13 +226,14 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
}
/**
- * This method will be used to get the sorted row
+ * This method will be used to get the sorted sort temp row from the sort temp files
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
public Object[] next() {
- return getSortedRecordFromFile();
+ IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
+ return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
}
/**
@@ -241,8 +242,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
- Object[] row = null;
+ private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
+ IntermediateSortTempRow row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
index 57a19bd..c7efbd9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.sort.sortdata;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.ExecutorService;
@@ -32,12 +32,10 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -69,7 +67,8 @@ public class SortDataRows {
private Semaphore semaphore;
private SortParameters parameters;
-
+ private SortStepRowHandler sortStepRowHandler;
+ private ThreadLocal<ByteBuffer> rowBuffer;
private int sortBufferSize;
private SortIntermediateFileMerger intermediateFileMerger;
@@ -79,7 +78,7 @@ public class SortDataRows {
public SortDataRows(SortParameters parameters,
SortIntermediateFileMerger intermediateFileMerger) {
this.parameters = parameters;
-
+ this.sortStepRowHandler = new SortStepRowHandler(parameters);
this.intermediateFileMerger = intermediateFileMerger;
int batchSize = CarbonProperties.getInstance().getBatchSize();
@@ -87,6 +86,12 @@ public class SortDataRows {
this.sortBufferSize = Math.max(parameters.getSortBufferSize(), batchSize);
// observer of writing file in thread
this.threadStatusObserver = new ThreadStatusObserver();
+ this.rowBuffer = new ThreadLocal<ByteBuffer>() {
+ @Override protected ByteBuffer initialValue() {
+ byte[] backedArray = new byte[2 * 1024 * 1024];
+ return ByteBuffer.wrap(backedArray);
+ }
+ };
}
/**
@@ -130,8 +135,7 @@ public class SortDataRows {
semaphore.acquire();
dataSorterAndWriterExecutorService.execute(new DataSorterAndWriter(recordHolderListLocal));
} catch (InterruptedException e) {
- LOGGER.error(e,
- "exception occurred while trying to acquire a semaphore lock: ");
+ LOGGER.error(e, "exception occurred while trying to acquire a semaphore lock: ");
throw new CarbonSortKeyAndGroupByException(e);
}
// create the new holder Array
@@ -158,7 +162,7 @@ public class SortDataRows {
}
intermediateFileMerger.startMergingIfPossible();
Object[][] recordHolderListLocal = recordHolderList;
- sizeLeft = sortBufferSize - entryCount ;
+ sizeLeft = sortBufferSize - entryCount;
if (sizeLeft > 0) {
System.arraycopy(rowBatch, 0, recordHolderListLocal, entryCount, sizeLeft);
}
@@ -212,7 +216,6 @@ public class SortDataRows {
locationChosen + File.separator + parameters.getTableName() +
System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(recordHolderList, this.entryCount, file);
-
}
startFileBasedMerge();
@@ -220,7 +223,7 @@ public class SortDataRows {
}
/**
- * Below method will be used to write data to file
+ * Below method will be used to write data to sort temp file
*
* @throws CarbonSortKeyAndGroupByException problem while writing
*/
@@ -233,60 +236,9 @@ public class SortDataRows {
parameters.getFileWriteBufferSize(), parameters.getSortTempCompressorName());
// write number of entries to the file
stream.writeInt(entryCountLocal);
- int complexDimColCount = parameters.getComplexDimColCount();
- int dimColCount = parameters.getDimColCount() + complexDimColCount;
- DataType[] type = parameters.getMeasureDataType();
- boolean[] noDictionaryDimnesionMapping = parameters.getNoDictionaryDimnesionColumn();
- Object[] row = null;
for (int i = 0; i < entryCountLocal; i++) {
- // get row from record holder list
- row = recordHolderList[i];
- int dimCount = 0;
- // write dictionary and non dictionary dimensions here.
- for (; dimCount < noDictionaryDimnesionMapping.length; dimCount++) {
- if (noDictionaryDimnesionMapping[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- stream.writeShort(col.length);
- stream.write(col);
- } else {
- stream.writeInt((int)row[dimCount]);
- }
- }
- // write complex dimensions here.
- for (; dimCount < dimColCount; dimCount++) {
- byte[] value = (byte[])row[dimCount];
- stream.writeShort(value.length);
- stream.write(value);
- }
- // as measures are stored in separate array.
- for (int mesCount = 0;
- mesCount < parameters.getMeasureColCount(); mesCount++) {
- Object value = row[mesCount + dimColCount];
- if (null != value) {
- stream.write((byte) 1);
- DataType dataType = type[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- stream.writeBoolean((boolean) value);
- } else if (dataType == DataTypes.SHORT) {
- stream.writeShort((Short) value);
- } else if (dataType == DataTypes.INT) {
- stream.writeInt((Integer) value);
- } else if (dataType == DataTypes.LONG) {
- stream.writeLong((Long) value);
- } else if (dataType == DataTypes.DOUBLE) {
- stream.writeDouble((Double) value);
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal val = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
- stream.writeInt(bigDecimalInBytes.length);
- stream.write(bigDecimalInBytes);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + type[mesCount]);
- }
- } else {
- stream.write((byte) 0);
- }
- }
+ sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToOutputStream(
+ recordHolderList[i], stream, rowBuffer.get());
}
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
@@ -301,7 +253,7 @@ public class SortDataRows {
*
* @throws CarbonSortKeyAndGroupByException
*/
- public void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
+ private void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
CarbonDataProcessorUtil.deleteSortLocationIfExists(parameters.getTempFileLocation());
}
@@ -380,7 +332,8 @@ public class SortDataRows {
// intermediate merging of sort temp files will be triggered
intermediateFileMerger.addFileToMerge(sortTempFile);
LOGGER.info("Time taken to sort and write sort temp file " + sortTempFile + " is: " + (
- System.currentTimeMillis() - startTime));
+ System.currentTimeMillis() - startTime) + ", sort temp file size in MB is "
+ + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
} catch (Throwable e) {
try {
threadStatusObserver.notifyFailed(e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index d726539..7e221a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -21,6 +21,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Comparator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -30,14 +31,11 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
+import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHolder> {
@@ -71,20 +69,13 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* return row
*/
- private Object[] returnRow;
- private int dimCnt;
- private int noDictDimCnt;
- private int complexCnt;
- private int measureCnt;
- private boolean[] isNoDictionaryDimensionColumn;
- private boolean[] isNoDictionarySortColumn;
- private DataType[] measureDataTypes;
+ private IntermediateSortTempRow returnRow;
private int readBufferSize;
private String compressorName;
- private Object[][] currentBuffer;
+ private IntermediateSortTempRow[] currentBuffer;
- private Object[][] backupBuffer;
+ private IntermediateSortTempRow[] backupBuffer;
private boolean isBackupFilled;
@@ -104,7 +95,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
* totalRecordFetch
*/
private int totalRecordFetch;
-
+ private TableFieldStat tableFieldStat;
+ private SortStepRowHandler sortStepRowHandler;
+ private Comparator<IntermediateSortTempRow> comparator;
/**
* Constructor to initialize
*
@@ -115,16 +108,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
public SortTempFileChunkHolder(File tempFile, SortParameters sortParameters, String tableName) {
// set temp file
this.tempFile = tempFile;
- this.dimCnt = sortParameters.getDimColCount();
- this.noDictDimCnt = sortParameters.getNoDictionaryCount();
- this.complexCnt = sortParameters.getComplexDimColCount();
- this.measureCnt = sortParameters.getMeasureColCount();
- this.isNoDictionaryDimensionColumn = sortParameters.getNoDictionaryDimnesionColumn();
- this.isNoDictionarySortColumn = sortParameters.getNoDictionarySortColumn();
- this.measureDataTypes = sortParameters.getMeasureDataType();
this.readBufferSize = sortParameters.getBufferSize();
this.compressorName = sortParameters.getSortTempCompressorName();
-
+ this.tableFieldStat = new TableFieldStat(sortParameters);
+ this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
+ this.comparator = new IntermediateSortTempRowComparator(
+ tableFieldStat.getIsSortColNoDictFlags());
this.executorService = Executors
.newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName));
}
@@ -178,7 +167,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
if (prefetch) {
fillDataForPrefetch();
} else {
- this.returnRow = getRowFromStream();
+ try {
+ this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ this.numberOfObjectRead++;
+ } catch (IOException e) {
+ throw new CarbonSortKeyAndGroupByException("Problem while reading rows", e);
+ }
}
}
@@ -212,86 +206,28 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
/**
- * Reads row from file
+ * Read a batch of row from stream
+ *
* @return Object[]
- * @throws CarbonSortKeyAndGroupByException
+ * @throws IOException if error occurs while reading from stream
*/
- private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
-
- Object[] holder = new Object[3];
- int index = 0;
- int nonDicIndex = 0;
- int[] dim = new int[dimCnt - noDictDimCnt];
- byte[][] nonDicArray = new byte[noDictDimCnt + complexCnt][];
- Object[] measures = new Object[measureCnt];
- try {
- // read dimension values
- for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
- if (isNoDictionaryDimensionColumn[i]) {
- short len = stream.readShort();
- byte[] array = new byte[len];
- stream.readFully(array);
- nonDicArray[nonDicIndex++] = array;
- } else {
- dim[index++] = stream.readInt();
- }
- }
-
- for (int i = 0; i < complexCnt; i++) {
- short len = stream.readShort();
- byte[] array = new byte[len];
- stream.readFully(array);
- nonDicArray[nonDicIndex++] = array;
- }
-
- index = 0;
- // read measure values
- for (int i = 0; i < measureCnt; i++) {
- if (stream.readByte() == 1) {
- DataType dataType = measureDataTypes[i];
- if (dataType == DataTypes.BOOLEAN) {
- measures[index++] = stream.readBoolean();
- } else if (dataType == DataTypes.SHORT) {
- measures[index++] = stream.readShort();
- } else if (dataType == DataTypes.INT) {
- measures[index++] = stream.readInt();
- } else if (dataType == DataTypes.LONG) {
- measures[index++] = stream.readLong();
- } else if (dataType == DataTypes.DOUBLE) {
- measures[index++] = stream.readDouble();
- } else if (DataTypes.isDecimal(dataType)) {
- int len = stream.readInt();
- byte[] buff = new byte[len];
- stream.readFully(buff);
- measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
- } else {
- throw new IllegalArgumentException("unsupported data type:" + dataType);
- }
- } else {
- measures[index++] = null;
- }
- }
-
- NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
-
- // increment number if record read
- this.numberOfObjectRead++;
- } catch (IOException e) {
- LOGGER.error("Problme while reading the madkey fom sort temp file");
- throw new CarbonSortKeyAndGroupByException("Problem while reading the sort temp file ", e);
+ private IntermediateSortTempRow[] readBatchedRowFromStream(int expected) throws IOException {
+ IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
+ for (int i = 0; i < expected; i++) {
+ IntermediateSortTempRow holder
+ = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
+ holders[i] = holder;
}
-
- //return out row
- return holder;
+ this.numberOfObjectRead += expected;
+ return holders;
}
/**
- * below method will be used to get the row
+ * below method will be used to get the sort temp row
*
* @return row
*/
- public Object[] getRow() {
+ public IntermediateSortTempRow getRow() {
return this.returnRow;
}
@@ -330,31 +266,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
@Override public int compareTo(SortTempFileChunkHolder other) {
- int diff = 0;
- int index = 0;
- int noDictionaryIndex = 0;
- int[] leftMdkArray = (int[]) returnRow[0];
- int[] rightMdkArray = (int[]) other.returnRow[0];
- byte[][] leftNonDictArray = (byte[][]) returnRow[1];
- byte[][] rightNonDictArray = (byte[][]) other.returnRow[1];
- for (boolean isNoDictionary : isNoDictionarySortColumn) {
- if (isNoDictionary) {
- diff = UnsafeComparer.INSTANCE
- .compareTo(leftNonDictArray[noDictionaryIndex], rightNonDictArray[noDictionaryIndex]);
- if (diff != 0) {
- return diff;
- }
- noDictionaryIndex++;
- } else {
- diff = leftMdkArray[index] - rightMdkArray[index];
- if (diff != 0) {
- return diff;
- }
- index++;
- }
-
- }
- return diff;
+ return comparator.compare(returnRow, other.getRow());
}
@Override public boolean equals(Object obj) {
@@ -372,9 +284,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
@Override public int hashCode() {
int hash = 0;
- hash += 31 * measureCnt;
- hash += 31 * dimCnt;
- hash += 31 * complexCnt;
+ hash += tableFieldStat.hashCode();
hash += tempFile.hashCode();
return hash;
}
@@ -414,16 +324,12 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords
- * @return
- * @throws CarbonSortKeyAndGroupByException
+ * @param numberOfRecords number of records to be read
+ * @return batch of intermediate sort temp row
+ * @throws IOException if error occurs while reading reading records
*/
- private Object[][] prefetchRecordsFromFile(int numberOfRecords)
- throws CarbonSortKeyAndGroupByException {
- Object[][] records = new Object[numberOfRecords][];
- for (int i = 0; i < numberOfRecords; i++) {
- records[i] = getRowFromStream();
- }
- return records;
+ private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
+ throws IOException {
+ return readBatchedRowFromStream(numberOfRecords);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
new file mode 100644
index 0000000..0d1303a
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * This class is used to hold field information for a table during data loading. These information
+ * will be used to convert/construct/destruct row in sort process step. Because complex field is
+ * processed the same as no-dict-no-sort-simple-dimension, so we treat them as the same and use
+ * `no-dict-no-sort-dim` related variable to represent them in this class.
+ */
+public class TableFieldStat implements Serializable {
+ private static final long serialVersionUID = 201712070950L;
+ private int dictSortDimCnt = 0;
+ private int dictNoSortDimCnt = 0;
+ private int noDictSortDimCnt = 0;
+ private int noDictNoSortDimCnt = 0;
+ // whether sort column is of dictionary type or not
+ private boolean[] isSortColNoDictFlags;
+ private int measureCnt;
+ private DataType[] measureDataType;
+
+ // indices for dict & sort dimension columns
+ private int[] dictSortDimIdx;
+ // indices for dict & no-sort dimension columns
+ private int[] dictNoSortDimIdx;
+ // indices for no-dict & sort dimension columns
+ private int[] noDictSortDimIdx;
+ // indices for no-dict & no-sort dimension columns, including complex columns
+ private int[] noDictNoSortDimIdx;
+ // indices for measure columns
+ private int[] measureIdx;
+
+ public TableFieldStat(SortParameters sortParameters) {
+ int noDictDimCnt = sortParameters.getNoDictionaryCount();
+ int complexDimCnt = sortParameters.getComplexDimColCount();
+ int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;
+ this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();
+ int sortColCnt = isSortColNoDictFlags.length;
+ for (boolean flag : isSortColNoDictFlags) {
+ if (flag) {
+ noDictSortDimCnt++;
+ } else {
+ dictSortDimCnt++;
+ }
+ }
+ this.measureCnt = sortParameters.getMeasureColCount();
+ this.measureDataType = sortParameters.getMeasureDataType();
+
+ // be careful that the default value is 0
+ this.dictSortDimIdx = new int[dictSortDimCnt];
+ this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];
+ this.noDictSortDimIdx = new int[noDictSortDimCnt];
+ this.noDictNoSortDimIdx = new int[noDictDimCnt + complexDimCnt - noDictSortDimCnt];
+ this.measureIdx = new int[measureCnt];
+
+ int tmpNoDictSortCnt = 0;
+ int tmpNoDictNoSortCnt = 0;
+ int tmpDictSortCnt = 0;
+ int tmpDictNoSortCnt = 0;
+ boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();
+
+ for (int i = 0; i < isDimNoDictFlags.length; i++) {
+ if (isDimNoDictFlags[i]) {
+ if (i < sortColCnt && isSortColNoDictFlags[i]) {
+ noDictSortDimIdx[tmpNoDictSortCnt++] = i;
+ } else {
+ noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;
+ }
+ } else {
+ if (i < sortColCnt && !isSortColNoDictFlags[i]) {
+ dictSortDimIdx[tmpDictSortCnt++] = i;
+ } else {
+ dictNoSortDimIdx[tmpDictNoSortCnt++] = i;
+ }
+ }
+ }
+ dictNoSortDimCnt = tmpDictNoSortCnt;
+
+ int base = isDimNoDictFlags.length;
+ // adding complex dimension columns
+ for (int i = 0; i < complexDimCnt; i++) {
+ noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = base + i;
+ }
+ noDictNoSortDimCnt = tmpNoDictNoSortCnt;
+
+ base += complexDimCnt;
+ // indices for measure columns
+ for (int i = 0; i < measureCnt; i++) {
+ measureIdx[i] = base + i;
+ }
+ }
+
+ public int getDictSortDimCnt() {
+ return dictSortDimCnt;
+ }
+
+ public int getDictNoSortDimCnt() {
+ return dictNoSortDimCnt;
+ }
+
+ public int getNoDictSortDimCnt() {
+ return noDictSortDimCnt;
+ }
+
+ public int getNoDictNoSortDimCnt() {
+ return noDictNoSortDimCnt;
+ }
+
+ public boolean[] getIsSortColNoDictFlags() {
+ return isSortColNoDictFlags;
+ }
+
+ public int getMeasureCnt() {
+ return measureCnt;
+ }
+
+ public DataType[] getMeasureDataType() {
+ return measureDataType;
+ }
+
+ public int[] getDictSortDimIdx() {
+ return dictSortDimIdx;
+ }
+
+ public int[] getDictNoSortDimIdx() {
+ return dictNoSortDimIdx;
+ }
+
+ public int[] getNoDictSortDimIdx() {
+ return noDictSortDimIdx;
+ }
+
+ public int[] getNoDictNoSortDimIdx() {
+ return noDictNoSortDimIdx;
+ }
+
+ public int[] getMeasureIdx() {
+ return measureIdx;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof TableFieldStat)) return false;
+ TableFieldStat that = (TableFieldStat) o;
+ return dictSortDimCnt == that.dictSortDimCnt
+ && dictNoSortDimCnt == that.dictNoSortDimCnt
+ && noDictSortDimCnt == that.noDictSortDimCnt
+ && noDictNoSortDimCnt == that.noDictNoSortDimCnt
+ && measureCnt == that.measureCnt;
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,
+ noDictNoSortDimCnt, measureCnt);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/21704cf7/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 276c3fe..3302094 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -323,10 +323,10 @@ public final class CarbonLoaderUtil {
* @return boolean which determines whether status update is done or not.
* @throws IOException
*/
- public static boolean updateSegmentFile(String tablePath, String segmentId, String segmentFile)
+ private static boolean updateSegmentFile(String tablePath, String segmentId, String segmentFile)
throws IOException {
boolean status = false;
- String tableStatusPath = CarbonTablePath.getTableStatusPath(tablePath);
+ String tableStatusPath = CarbonTablePath.getTableStatusFilePath(tablePath);
String metadataPath = CarbonTablePath.getMetadataPath(tablePath);
AbsoluteTableIdentifier absoluteTableIdentifier =
AbsoluteTableIdentifier.from(tablePath, null, null);
[42/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
[CARBONDATA-2189] Add DataMapProvider developer interface
Add developer interface for 2 types of DataMap:
1.IndexDataMap: DataMap that leveraging index to accelerate filter query
2.MVDataMap: DataMap that leveraging Materialized View to accelerate olap style query, like SPJG query (select, predicate, join, groupby)
This PR adds support for following logic when creating and dropping the DataMap
This closes #1987
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89a12af5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89a12af5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89a12af5
Branch: refs/heads/master
Commit: 89a12af5aba17f12c4e695971982abfeff256fc1
Parents: 56330ae
Author: Jacky Li <ja...@qq.com>
Authored: Thu Feb 22 20:59:59 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../sql/MalformedDataMapCommandException.java | 4 +
.../carbondata/core/datamap/DataMapChooser.java | 5 +-
.../core/datamap/DataMapRegistry.java | 37 +
.../core/datamap/DataMapStoreManager.java | 82 +-
.../carbondata/core/datamap/TableDataMap.java | 42 +-
.../core/datamap/dev/AbstractDataMapWriter.java | 2 +-
.../carbondata/core/datamap/dev/DataMap.java | 56 --
.../core/datamap/dev/DataMapFactory.java | 86 --
.../core/datamap/dev/IndexDataMap.java | 56 ++
.../core/datamap/dev/IndexDataMapFactory.java | 86 ++
.../cgdatamap/AbstractCoarseGrainDataMap.java | 24 -
.../AbstractCoarseGrainDataMapFactory.java | 34 -
.../AbstractCoarseGrainIndexDataMap.java | 24 +
.../AbstractCoarseGrainIndexDataMapFactory.java | 34 +
.../dev/expr/DataMapExprWrapperImpl.java | 2 +-
.../dev/fgdatamap/AbstractFineGrainDataMap.java | 24 -
.../AbstractFineGrainDataMapFactory.java | 38 -
.../AbstractFineGrainIndexDataMap.java | 24 +
.../AbstractFineGrainIndexDataMapFactory.java | 38 +
.../indexstore/BlockletDataMapIndexStore.java | 33 +-
.../blockletindex/BlockletDataMap.java | 971 -------------------
.../blockletindex/BlockletDataMapFactory.java | 285 ------
.../blockletindex/BlockletDataMapModel.java | 2 +-
.../blockletindex/BlockletIndexDataMap.java | 971 +++++++++++++++++++
.../BlockletIndexDataMapFactory.java | 285 ++++++
.../core/metadata/schema/table/CarbonTable.java | 10 +-
.../metadata/schema/table/DataMapSchema.java | 1 +
.../schema/table/DataMapSchemaFactory.java | 13 +-
.../core/metadata/schema/table/TableSchema.java | 3 +-
.../blockletindex/TestBlockletDataMap.java | 66 --
.../blockletindex/TestBlockletIndexDataMap.java | 59 ++
.../datamap/examples/MinMaxDataMap.java | 152 ---
.../datamap/examples/MinMaxDataMapFactory.java | 117 ---
.../datamap/examples/MinMaxIndexDataMap.java | 150 +++
.../examples/MinMaxIndexDataMapFactory.java | 117 +++
.../MinMaxDataMapExample.scala | 4 +-
docs/datamap-developer-guide.md | 16 +
.../hadoop/api/CarbonTableInputFormat.java | 5 +-
.../preaggregate/TestPreAggCreateCommand.scala | 2 +-
.../TestPreAggregateTableSelection.scala | 4 +-
.../timeseries/TestTimeSeriesCreateTable.scala | 4 +-
.../testsuite/datamap/CGDataMapTestCase.scala | 381 --------
.../datamap/CGIndexDataMapTestCase.scala | 383 ++++++++
.../testsuite/datamap/DataMapWriterSuite.scala | 216 -----
.../testsuite/datamap/FGDataMapTestCase.scala | 473 ---------
.../datamap/FGIndexDataMapTestCase.scala | 472 +++++++++
.../datamap/IndexDataMapWriterSuite.scala | 217 +++++
.../testsuite/datamap/TestDataMapCommand.scala | 288 ------
.../datamap/TestIndexDataMapCommand.scala | 285 ++++++
.../carbondata/datamap/DataMapManager.java | 53 +
.../carbondata/datamap/DataMapProperty.java | 32 +
.../carbondata/datamap/DataMapProvider.java | 105 ++
.../datamap/IndexDataMapProvider.java | 116 +++
.../datamap/PreAggregateDataMapProvider.java | 92 ++
.../datamap/TimeseriesDataMapProvider.java | 50 +
.../scala/org/apache/spark/sql/CarbonEnv.scala | 2 +-
.../datamap/CarbonCreateDataMapCommand.scala | 92 +-
.../datamap/CarbonDropDataMapCommand.scala | 73 +-
.../CarbonAlterTableDropPartitionCommand.scala | 2 +-
.../CarbonAlterTableSplitPartitionCommand.scala | 2 +-
.../CreatePreAggregateTableCommand.scala | 203 ----
.../preaaggregate/PreAggregateTableHelper.scala | 195 ++++
.../preaaggregate/PreAggregateUtil.scala | 24 +-
.../CarbonAlterTableAddColumnCommand.scala | 2 +-
.../CarbonAlterTableDataTypeChangeCommand.scala | 2 +-
.../CarbonAlterTableDropColumnCommand.scala | 2 +-
.../schema/CarbonAlterTableRenameCommand.scala | 2 +-
.../command/timeseries/TimeSeriesUtil.scala | 16 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 3 +-
.../spark/sql/hive/CarbonHiveMetaStore.scala | 3 +-
.../apache/spark/sql/hive/CarbonMetaStore.scala | 5 +-
.../org/apache/spark/util/AlterTableUtil.scala | 10 +-
.../datamap/DataMapWriterListener.java | 8 +-
73 files changed, 4094 insertions(+), 3683 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
index 7c25b2c..83cae7c 100644
--- a/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
+++ b/common/src/main/java/org/apache/carbondata/common/exceptions/sql/MalformedDataMapCommandException.java
@@ -34,4 +34,8 @@ public class MalformedDataMapCommandException extends MalformedCarbonCommandExce
public MalformedDataMapCommandException(String msg) {
super(msg);
}
+
+ public MalformedDataMapCommandException(String msg, Throwable e) {
+ super(msg, e);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index 5155009..41e9b56 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -212,9 +212,10 @@ public class DataMapChooser {
List<ColumnExpression> columnExpressions, Set<ExpressionType> expressionTypes) {
List<DataMapTuple> tuples = new ArrayList<>();
for (TableDataMap dataMap : allDataMap) {
- if (contains(dataMap.getDataMapFactory().getMeta(), columnExpressions, expressionTypes)) {
+ if (contains(dataMap.getIndexDataMapFactory().getMeta(), columnExpressions, expressionTypes))
+ {
tuples.add(
- new DataMapTuple(dataMap.getDataMapFactory().getMeta().getIndexedColumns().size(),
+ new DataMapTuple(dataMap.getIndexDataMapFactory().getMeta().getIndexedColumns().size(),
dataMap));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
new file mode 100644
index 0000000..03c0c3e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapRegistry.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class DataMapRegistry {
+ private static Map<String, String> shortNameToClassName = new ConcurrentHashMap<>();
+
+ public static void registerDataMap(String datamapClassName, String shortName) {
+ Objects.requireNonNull(datamapClassName);
+ Objects.requireNonNull(shortName);
+ shortNameToClassName.put(shortName, datamapClassName);
+ }
+
+ public static String getDataMapClassName(String shortName) {
+ Objects.requireNonNull(shortName);
+ return shortNameToClassName.get(shortName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index cd89d74..ed4e18c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -26,11 +26,10 @@ import org.apache.carbondata.common.exceptions.MetadataProcessException;
import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -69,7 +68,7 @@ public final class DataMapStoreManager {
List<TableDataMap> tableDataMaps = getAllDataMap(carbonTable);
if (tableDataMaps != null) {
for (TableDataMap dataMap : tableDataMaps) {
- if (mapType == dataMap.getDataMapFactory().getDataMapType()) {
+ if (mapType == dataMap.getIndexDataMapFactory().getDataMapType()) {
dataMaps.add(dataMap);
}
}
@@ -87,8 +86,8 @@ public final class DataMapStoreManager {
List<TableDataMap> dataMaps = new ArrayList<>();
if (dataMapSchemaList != null) {
for (DataMapSchema dataMapSchema : dataMapSchemaList) {
- if (!dataMapSchema.getClassName()
- .equalsIgnoreCase(DataMapProvider.PREAGGREGATE.getClassName())) {
+ if (!dataMapSchema.getClassName().equalsIgnoreCase(
+ DataMapProvider.PREAGGREGATE.toString())) {
dataMaps.add(getDataMap(carbonTable.getAbsoluteTableIdentifier(), dataMapSchema));
}
}
@@ -97,13 +96,13 @@ public final class DataMapStoreManager {
}
/**
- * It gives the default datamap of the table. Default datamap of any table is BlockletDataMap
+ * It gives the default datamap of the table. Default datamap of any table is BlockletIndexDataMap
*
* @param identifier
* @return
*/
public TableDataMap getDefaultDataMap(AbsoluteTableIdentifier identifier) {
- return getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
+ return getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA);
}
/**
@@ -142,9 +141,26 @@ public final class DataMapStoreManager {
* Return a new datamap instance and registered in the store manager.
* The datamap is created using datamap name, datamap factory class and table identifier.
*/
- public TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
- DataMapSchema dataMapSchema)
- throws MalformedDataMapCommandException {
+ private TableDataMap createAndRegisterDataMap(AbsoluteTableIdentifier identifier,
+ DataMapSchema dataMapSchema) throws MalformedDataMapCommandException {
+ IndexDataMapFactory indexDataMapFactory;
+ try {
+ // try to create datamap by reflection to test whether it is a valid IndexDataMapFactory class
+ Class<? extends IndexDataMapFactory> factoryClass =
+ (Class<? extends IndexDataMapFactory>) Class.forName(dataMapSchema.getClassName());
+ indexDataMapFactory = factoryClass.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new MalformedDataMapCommandException(
+ "DataMap '" + dataMapSchema.getClassName() + "' not found");
+ } catch (Throwable e) {
+ throw new MetadataProcessException(
+ "failed to create DataMap '" + dataMapSchema.getClassName() + "'", e);
+ }
+ return registerDataMap(identifier, dataMapSchema, indexDataMapFactory);
+ }
+
+ public TableDataMap registerDataMap(AbsoluteTableIdentifier identifier,
+ DataMapSchema dataMapSchema, IndexDataMapFactory indexDataMapFactory) {
String table = identifier.getCarbonTableIdentifier().getTableUniqueName();
// Just update the segmentRefreshMap with the table if not added.
getTableSegmentRefresher(identifier);
@@ -152,37 +168,19 @@ public final class DataMapStoreManager {
if (tableDataMaps == null) {
tableDataMaps = new ArrayList<>();
}
- String dataMapName = dataMapSchema.getDataMapName();
- TableDataMap dataMap = getTableDataMap(dataMapName, tableDataMaps);
- if (dataMap != null && dataMap.getDataMapSchema().getDataMapName()
- .equalsIgnoreCase(dataMapName)) {
- throw new MalformedDataMapCommandException("Already datamap exists in that path with type " +
- dataMapName);
- }
- try {
- // try to create datamap by reflection to test whether it is a valid DataMapFactory class
- Class<? extends DataMapFactory> factoryClass =
- (Class<? extends DataMapFactory>) Class.forName(dataMapSchema.getClassName());
- DataMapFactory dataMapFactory = factoryClass.newInstance();
- dataMapFactory.init(identifier, dataMapSchema);
- BlockletDetailsFetcher blockletDetailsFetcher;
- SegmentPropertiesFetcher segmentPropertiesFetcher = null;
- if (dataMapFactory instanceof BlockletDetailsFetcher) {
- blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory;
- } else {
- blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
- }
- segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
- dataMap = new TableDataMap(identifier, dataMapSchema, dataMapFactory, blockletDetailsFetcher,
- segmentPropertiesFetcher);
- } catch (ClassNotFoundException e) {
- throw new MalformedDataMapCommandException("DataMap class '" +
- dataMapSchema.getClassName() + "' not found");
- } catch (Throwable e) {
- throw new MetadataProcessException(
- "failed to create DataMap instance for '" + dataMapSchema.getClassName() + "'", e);
+ indexDataMapFactory.init(identifier, dataMapSchema);
+ BlockletDetailsFetcher blockletDetailsFetcher;
+ SegmentPropertiesFetcher segmentPropertiesFetcher = null;
+ if (indexDataMapFactory instanceof BlockletDetailsFetcher) {
+ blockletDetailsFetcher = (BlockletDetailsFetcher) indexDataMapFactory;
+ } else {
+ blockletDetailsFetcher = getBlockletDetailsFetcher(identifier);
}
+ segmentPropertiesFetcher = (SegmentPropertiesFetcher) blockletDetailsFetcher;
+ TableDataMap dataMap = new TableDataMap(identifier, dataMapSchema, indexDataMapFactory,
+ blockletDetailsFetcher, segmentPropertiesFetcher);
+
tableDataMaps.add(dataMap);
allDataMaps.put(table, tableDataMaps);
return dataMap;
@@ -262,8 +260,8 @@ public final class DataMapStoreManager {
* @return
*/
private BlockletDetailsFetcher getBlockletDetailsFetcher(AbsoluteTableIdentifier identifier) {
- TableDataMap blockletMap = getDataMap(identifier, BlockletDataMapFactory.DATA_MAP_SCHEMA);
- return (BlockletDetailsFetcher) blockletMap.getDataMapFactory();
+ TableDataMap blockletMap = getDataMap(identifier, BlockletIndexDataMapFactory.DATA_MAP_SCHEMA);
+ return (BlockletDetailsFetcher) blockletMap.getIndexDataMapFactory();
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index 3c66c89..b9a50d2 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -22,8 +22,8 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.dev.BlockletSerializer;
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
+import org.apache.carbondata.core.datamap.dev.IndexDataMap;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.indexstore.Blocklet;
@@ -40,7 +40,7 @@ import org.apache.carbondata.events.OperationContext;
import org.apache.carbondata.events.OperationEventListener;
/**
- * DataMap at the table level, user can add any number of datamaps for one table. Depends
+ * IndexDataMap at the table level, user can add any number of datamaps for one table. Depends
* on the filter condition it can prune the blocklets.
*/
public final class TableDataMap extends OperationEventListener {
@@ -49,7 +49,7 @@ public final class TableDataMap extends OperationEventListener {
private DataMapSchema dataMapSchema;
- private DataMapFactory dataMapFactory;
+ private IndexDataMapFactory indexDataMapFactory;
private BlockletDetailsFetcher blockletDetailsFetcher;
@@ -59,11 +59,11 @@ public final class TableDataMap extends OperationEventListener {
* It is called to initialize and load the required table datamap metadata.
*/
public TableDataMap(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema,
- DataMapFactory dataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
+ IndexDataMapFactory indexDataMapFactory, BlockletDetailsFetcher blockletDetailsFetcher,
SegmentPropertiesFetcher segmentPropertiesFetcher) {
this.identifier = identifier;
this.dataMapSchema = dataMapSchema;
- this.dataMapFactory = dataMapFactory;
+ this.indexDataMapFactory = indexDataMapFactory;
this.blockletDetailsFetcher = blockletDetailsFetcher;
this.segmentPropertiesFetcher = segmentPropertiesFetcher;
}
@@ -85,9 +85,9 @@ public final class TableDataMap extends OperationEventListener {
if (filterExp == null) {
pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions);
} else {
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
+ List<IndexDataMap> dataMaps = indexDataMapFactory.getDataMaps(segment);
segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment);
- for (DataMap dataMap : dataMaps) {
+ for (IndexDataMap dataMap : dataMaps) {
pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
}
}
@@ -115,7 +115,7 @@ public final class TableDataMap extends OperationEventListener {
public List<DataMapDistributable> toDistributable(List<Segment> segments) throws IOException {
List<DataMapDistributable> distributables = new ArrayList<>();
for (Segment segment : segments) {
- List<DataMapDistributable> list = dataMapFactory.toDistributable(segment);
+ List<DataMapDistributable> list = indexDataMapFactory.toDistributable(segment);
for (DataMapDistributable distributable: list) {
distributable.setDataMapSchema(dataMapSchema);
distributable.setSegment(segment);
@@ -138,10 +138,10 @@ public final class TableDataMap extends OperationEventListener {
FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
List<Blocklet> blocklets = new ArrayList<>();
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
- for (DataMap dataMap : dataMaps) {
+ List<IndexDataMap> indexDataMaps = indexDataMapFactory.getDataMaps(distributable);
+ for (IndexDataMap indexDataMap : indexDataMaps) {
blocklets.addAll(
- dataMap.prune(
+ indexDataMap.prune(
filterExp,
segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment()),
partitions));
@@ -150,13 +150,13 @@ public final class TableDataMap extends OperationEventListener {
String writePath =
identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapSchema
.getDataMapName();
- if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ if (indexDataMapFactory.getDataMapType() == DataMapType.FG) {
FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
}
for (Blocklet blocklet : blocklets) {
ExtendedBlocklet detailedBlocklet =
blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegment());
- if (dataMapFactory.getDataMapType() == DataMapType.FG) {
+ if (indexDataMapFactory.getDataMapType() == DataMapType.FG) {
String blockletwritePath =
writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
detailedBlocklet.setDataMapWriterPath(blockletwritePath);
@@ -174,7 +174,7 @@ public final class TableDataMap extends OperationEventListener {
*/
public void clear(List<Segment> segments) {
for (Segment segment: segments) {
- dataMapFactory.clear(segment);
+ indexDataMapFactory.clear(segment);
}
}
@@ -182,19 +182,19 @@ public final class TableDataMap extends OperationEventListener {
* Clears all datamap
*/
public void clear() {
- dataMapFactory.clear();
+ indexDataMapFactory.clear();
}
public DataMapSchema getDataMapSchema() {
return dataMapSchema;
}
- public DataMapFactory getDataMapFactory() {
- return dataMapFactory;
+ public IndexDataMapFactory getIndexDataMapFactory() {
+ return indexDataMapFactory;
}
@Override public void onEvent(Event event, OperationContext opContext) throws Exception {
- dataMapFactory.fireEvent(event);
+ indexDataMapFactory.fireEvent(event);
}
/**
@@ -209,8 +209,8 @@ public final class TableDataMap extends OperationEventListener {
throws IOException {
List<Segment> prunedSegments = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (Segment segment : segments) {
- List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
- for (DataMap dataMap : dataMaps) {
+ List<IndexDataMap> dataMaps = indexDataMapFactory.getDataMaps(segment);
+ for (IndexDataMap dataMap : dataMaps) {
if (dataMap.isScanRequired(filterExp)) {
// If any one task in a given segment contains the data that means the segment need to
// be scanned and we need to validate further data maps in the same segment
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
index de6dcb1..c6961c7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -71,7 +71,7 @@ public abstract class AbstractDataMapWriter {
/**
* Add the column pages row to the datamap, order of pages is same as `indexColumns` in
- * DataMapMeta returned in DataMapFactory.
+ * DataMapMeta returned in IndexDataMapFactory.
* Implementation should copy the content of `pages` as needed, because `pages` memory
* may be freed after this method returns, if using unsafe column page.
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
deleted file mode 100644
index dd5507c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMap.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-
-/**
- * Datamap is an entity which can store and retrieve index data.
- */
-public interface DataMap<T extends Blocklet> {
-
- /**
- * It is called to load the data map to memory or to initialize it.
- */
- void init(DataMapModel dataMapModel) throws MemoryException, IOException;
-
- /**
- * Prune the datamap with filter expression and partition information. It returns the list of
- * blocklets where these filters can exist.
- */
- List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions);
-
- // TODO Move this method to Abstract class
- /**
- * Validate whether the current segment needs to be fetching the required data
- */
- boolean isScanRequired(FilterResolverIntf filterExp);
-
- /**
- * Clear complete index table and release memory.
- */
- void clear();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
deleted file mode 100644
index b59de81..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.events.Event;
-
-/**
- * Interface for datamap factory, it is responsible for creating the datamap.
- */
-public interface DataMapFactory<T extends DataMap> {
-
- /**
- * Initialization of Datamap factory with the identifier and datamap name
- */
- void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
-
- /**
- * Return a new write for this datamap
- */
- AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath);
-
- /**
- * Get the datamap for segmentid
- */
- List<T> getDataMaps(Segment segment) throws IOException;
-
- /**
- * Get datamaps for distributable object.
- */
- List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
-
- /**
- * Get all distributable objects of a segmentid
- * @return
- */
- List<DataMapDistributable> toDistributable(Segment segment);
-
- /**
- *
- * @param event
- */
- void fireEvent(Event event);
-
- /**
- * Clears datamap of the segment
- */
- void clear(Segment segment);
-
- /**
- * Clear all datamaps from memory
- */
- void clear();
-
- /**
- * Return metadata of this datamap
- */
- DataMapMeta getMeta();
-
- /**
- * Type of datamap whether it is FG or CG
- */
- DataMapType getDataMapType();
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
new file mode 100644
index 0000000..eed3ac9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMap.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * Datamap is an entity which can store and retrieve index data.
+ */
+public interface IndexDataMap<T extends Blocklet> {
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ void init(DataMapModel dataMapModel) throws MemoryException, IOException;
+
+ /**
+ * Prune the datamap with filter expression and partition information. It returns the list of
+ * blocklets where these filters can exist.
+ */
+ List<T> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<PartitionSpec> partitions);
+
+ // TODO Move this method to Abstract class
+ /**
+ * Validate whether the current segment needs to be fetching the required data
+ */
+ boolean isScanRequired(FilterResolverIntf filterExp);
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ void clear();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
new file mode 100644
index 0000000..12c9fd9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/IndexDataMapFactory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.events.Event;
+
+/**
+ * Interface for datamap factory, it is responsible for creating the datamap.
+ */
+public interface IndexDataMapFactory<T extends IndexDataMap> {
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema);
+
+ /**
+ * Return a new write for this datamap
+ */
+ AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath);
+
+ /**
+ * Get the datamap for segmentid
+ */
+ List<T> getDataMaps(Segment segment) throws IOException;
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ List<T> getDataMaps(DataMapDistributable distributable) throws IOException;
+
+ /**
+ * Get all distributable objects of a segmentid
+ * @return
+ */
+ List<DataMapDistributable> toDistributable(Segment segment);
+
+ /**
+ *
+ * @param event
+ */
+ void fireEvent(Event event);
+
+ /**
+ * Clears datamap of the segment
+ */
+ void clear(Segment segment);
+
+ /**
+ * Clear all datamaps from memory
+ */
+ void clear();
+
+ /**
+ * Return metadata of this datamap
+ */
+ DataMapMeta getMeta();
+
+ /**
+ * Type of datamap whether it is FG or CG
+ */
+ DataMapType getDataMapType();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
deleted file mode 100644
index d79d0c6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMap.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.cgdatamap;
-
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.indexstore.Blocklet;
-
-public abstract class AbstractCoarseGrainDataMap implements DataMap<Blocklet> {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
deleted file mode 100644
index 9789992..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainDataMapFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.cgdatamap;
-
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-
-/**
- * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
- * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
- * information of block and blocklet.
- * 3. Based on the splits scanrdd schedule the tasks.
- */
-public abstract class AbstractCoarseGrainDataMapFactory
- implements DataMapFactory<AbstractCoarseGrainDataMap> {
-
- @Override public DataMapType getDataMapType() {
- return DataMapType.CG;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
new file mode 100644
index 0000000..df9d4e8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.IndexDataMap;
+import org.apache.carbondata.core.indexstore.Blocklet;
+
+public abstract class AbstractCoarseGrainIndexDataMap implements IndexDataMap<Blocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
new file mode 100644
index 0000000..037c32e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/cgdatamap/AbstractCoarseGrainIndexDataMapFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.cgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of CGdatamap.
+ * 2. The prune method of CGDatamap return list Blocklet , these blocklets contain the
+ * information of block and blocklet.
+ * 3. Based on the splits scanrdd schedule the tasks.
+ */
+public abstract class AbstractCoarseGrainIndexDataMapFactory
+ implements IndexDataMapFactory<AbstractCoarseGrainIndexDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.CG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
index fac6cc1..312fc11 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
@@ -83,6 +83,6 @@ public class DataMapExprWrapperImpl implements DataMapExprWrapper {
}
@Override public DataMapType getDataMapType() {
- return dataMap.getDataMapFactory().getDataMapType();
+ return dataMap.getIndexDataMapFactory().getDataMapType();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
deleted file mode 100644
index 310fb3b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMap.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.fgdatamap;
-
-import org.apache.carbondata.core.datamap.dev.DataMap;
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
-
-public abstract class AbstractFineGrainDataMap implements DataMap<FineGrainBlocklet> {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
deleted file mode 100644
index 1ca7fc3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainDataMapFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datamap.dev.fgdatamap;
-
-import org.apache.carbondata.core.datamap.DataMapType;
-import org.apache.carbondata.core.datamap.dev.DataMapFactory;
-
-/**
- * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
- * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
- * information of block, blocklet, page and rowids information as well.
- * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
- * blocklet and filepath information as part of Splits.
- * 4. Based on the splits scanrdd schedule the tasks.
- * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
- * bitset if exists. And pass this bitset as input to it.
- */
-public abstract class AbstractFineGrainDataMapFactory
- implements DataMapFactory<AbstractFineGrainDataMap> {
-
- @Override public DataMapType getDataMapType() {
- return DataMapType.FG;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
new file mode 100644
index 0000000..ea536b9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMap.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.dev.IndexDataMap;
+import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
+
+public abstract class AbstractFineGrainIndexDataMap implements IndexDataMap<FineGrainBlocklet> {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
new file mode 100644
index 0000000..762c233
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/fgdatamap/AbstractFineGrainIndexDataMapFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datamap.dev.fgdatamap;
+
+import org.apache.carbondata.core.datamap.DataMapType;
+import org.apache.carbondata.core.datamap.dev.IndexDataMapFactory;
+
+/**
+ * 1. Any filter query which hits the table with datamap will call prune method of FGdatamap.
+ * 2. The prune method of FGDatamap return list FineGrainBlocklet , these blocklets contain the
+ * information of block, blocklet, page and rowids information as well.
+ * 3. The pruned blocklets are internally wriitten to file and returns only the block ,
+ * blocklet and filepath information as part of Splits.
+ * 4. Based on the splits scanrdd schedule the tasks.
+ * 5. In filterscanner we check the datamapwriterpath from split and reNoteads the
+ * bitset if exists. And pass this bitset as input to it.
+ */
+public abstract class AbstractFineGrainIndexDataMapFactory
+ implements IndexDataMapFactory<AbstractFineGrainIndexDataMap> {
+
+ @Override public DataMapType getDataMapType() {
+ return DataMapType.FG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index defaf39..67dbe88 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -31,8 +31,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapModel;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMap;
import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
@@ -47,7 +47,7 @@ import org.apache.hadoop.fs.RemoteIterator;
* blocks
*/
public class BlockletDataMapIndexStore
- implements Cache<TableBlockIndexUniqueIdentifier, BlockletDataMap> {
+ implements Cache<TableBlockIndexUniqueIdentifier, BlockletIndexDataMap> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockletDataMapIndexStore.class.getName());
/**
@@ -74,10 +74,10 @@ public class BlockletDataMapIndexStore
}
@Override
- public BlockletDataMap get(TableBlockIndexUniqueIdentifier identifier)
+ public BlockletIndexDataMap get(TableBlockIndexUniqueIdentifier identifier)
throws IOException {
String lruCacheKey = identifier.getUniqueTableSegmentIdentifier();
- BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
+ BlockletIndexDataMap dataMap = (BlockletIndexDataMap) lruCache.get(lruCacheKey);
if (dataMap == null) {
try {
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
@@ -134,14 +134,15 @@ public class BlockletDataMapIndexStore
}
@Override
- public List<BlockletDataMap> getAll(
+ public List<BlockletIndexDataMap> getAll(
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
- List<BlockletDataMap> blockletDataMaps = new ArrayList<>(tableSegmentUniqueIdentifiers.size());
+ List<BlockletIndexDataMap> blockletDataMaps =
+ new ArrayList<>(tableSegmentUniqueIdentifiers.size());
List<TableBlockIndexUniqueIdentifier> missedIdentifiers = new ArrayList<>();
// Get the datamaps for each indexfile from cache.
try {
for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
- BlockletDataMap ifPresent = getIfPresent(identifier);
+ BlockletIndexDataMap ifPresent = getIfPresent(identifier);
if (ifPresent != null) {
blockletDataMaps.add(ifPresent);
} else {
@@ -159,7 +160,7 @@ public class BlockletDataMapIndexStore
}
}
} catch (Throwable e) {
- for (BlockletDataMap dataMap : blockletDataMaps) {
+ for (BlockletIndexDataMap dataMap : blockletDataMaps) {
dataMap.clear();
}
throw new IOException("Problem in loading segment blocks.", e);
@@ -174,9 +175,9 @@ public class BlockletDataMapIndexStore
* @return
*/
@Override
- public BlockletDataMap getIfPresent(
+ public BlockletIndexDataMap getIfPresent(
TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
- return (BlockletDataMap) lruCache.get(
+ return (BlockletIndexDataMap) lruCache.get(
tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
}
@@ -199,7 +200,7 @@ public class BlockletDataMapIndexStore
* @return map of taks id to segment mapping
* @throws IOException
*/
- private BlockletDataMap loadAndGetDataMap(
+ private BlockletIndexDataMap loadAndGetDataMap(
TableBlockIndexUniqueIdentifier identifier,
SegmentIndexFileStore indexFileStore,
Map<String, BlockMetaInfo> blockMetaInfoMap)
@@ -210,9 +211,9 @@ public class BlockletDataMapIndexStore
if (lock == null) {
lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
}
- BlockletDataMap dataMap;
+ BlockletIndexDataMap dataMap;
synchronized (lock) {
- dataMap = new BlockletDataMap();
+ dataMap = new BlockletIndexDataMap();
dataMap.init(new BlockletDataMapModel(
identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()),
@@ -248,9 +249,9 @@ public class BlockletDataMapIndexStore
@Override
public void clearAccessCount(
List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) {
- for (TableBlockIndexUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) {
- BlockletDataMap cacheable =
- (BlockletDataMap) lruCache.get(segmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
+ BlockletIndexDataMap cacheable =
+ (BlockletIndexDataMap) lruCache.get(identifier.getUniqueTableSegmentIdentifier());
cacheable.clear();
}
}
[20/54] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad]
Add size base block allocation in data loading
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
new file mode 100644
index 0000000..0ae0b93
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+
+import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+import org.apache.carbondata.core.util.NonDictionaryUtil;
+
+public class RowComparator implements Comparator<Object[]> {
+ /**
+ * noDictionaryCount represent number of no dictionary cols
+ */
+ private int noDictionaryCount;
+
+ /**
+ * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
+ */
+ private boolean[] noDictionarySortColumnMaping;
+
+ /**
+ * @param noDictionarySortColumnMaping
+ * @param noDictionaryCount
+ */
+ public RowComparator(boolean[] noDictionarySortColumnMaping, int noDictionaryCount) {
+ this.noDictionaryCount = noDictionaryCount;
+ this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
+ }
+
+ /**
+ * Below method will be used to compare two mdkey
+ */
+ public int compare(Object[] rowA, Object[] rowB) {
+ int diff = 0;
+
+ int normalIndex = 0;
+ int noDictionaryindex = 0;
+
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
+
+ if (isNoDictionary) {
+ byte[] byteArr1 = (byte[]) rowA[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+
+ ByteBuffer buff1 = ByteBuffer.wrap(byteArr1);
+
+ // extract a high card dims from complete byte[].
+ NonDictionaryUtil
+ .extractSingleHighCardDims(byteArr1, noDictionaryindex, noDictionaryCount, buff1);
+
+ byte[] byteArr2 = (byte[]) rowB[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+
+ ByteBuffer buff2 = ByteBuffer.wrap(byteArr2);
+
+ // extract a high card dims from complete byte[].
+ NonDictionaryUtil
+ .extractSingleHighCardDims(byteArr2, noDictionaryindex, noDictionaryCount, buff2);
+
+ int difference = UnsafeComparer.INSTANCE.compareTo(buff1, buff2);
+ if (difference != 0) {
+ return difference;
+ }
+ noDictionaryindex++;
+ } else {
+ int dimFieldA = NonDictionaryUtil.getDimension(normalIndex, rowA);
+ int dimFieldB = NonDictionaryUtil.getDimension(normalIndex, rowB);
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ normalIndex++;
+ }
+
+ }
+
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
new file mode 100644
index 0000000..0883ae1
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/RowComparatorForNormalDims.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.sort.sortdata;
+
+import java.util.Comparator;
+
+import org.apache.carbondata.core.util.NonDictionaryUtil;
+
+/**
+ * This class is used as comparator for comparing dims which are non high cardinality dims.
+ * Here the dims will be in form of int[] (surrogates) so directly comparing the integers.
+ */
+public class RowComparatorForNormalDims implements Comparator<Object[]> {
+ /**
+ * dimension count
+ */
+ private int numberOfSortColumns;
+
+ /**
+ * RowComparatorForNormalDims Constructor
+ *
+ * @param numberOfSortColumns
+ */
+ public RowComparatorForNormalDims(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
+ }
+
+ /**
+ * Below method will be used to compare two surrogate keys
+ *
+ * @see Comparator#compare(Object, Object)
+ */
+ public int compare(Object[] rowA, Object[] rowB) {
+ int diff = 0;
+
+ for (int i = 0; i < numberOfSortColumns; i++) {
+
+ int dimFieldA = NonDictionaryUtil.getDimension(i, rowA);
+ int dimFieldB = NonDictionaryUtil.getDimension(i, rowB);
+
+ diff = dimFieldA - dimFieldB;
+ if (diff != 0) {
+ return diff;
+ }
+ }
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
index a4ac0ea..88695b9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SingleThreadFinalSortFilesMerger.java
@@ -37,8 +37,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -73,12 +71,12 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* tableName
*/
private String tableName;
- private SortParameters sortParameters;
- private SortStepRowHandler sortStepRowHandler;
+
/**
* tempFileLocation
*/
private String[] tempFileLocation;
+ private SortParameters sortParameters;
private int maxThreadForSorting;
@@ -91,7 +89,6 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.sortParameters = sortParameters;
- this.sortStepRowHandler = new SortStepRowHandler(sortParameters);
try {
maxThreadForSorting = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD,
@@ -110,7 +107,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
public void startFinalMerge() throws CarbonDataWriterException {
List<File> filesToMerge = getFilesToMergeSort();
- if (filesToMerge.size() == 0) {
+ if (filesToMerge.size() == 0)
+ {
LOGGER.info("No file to merge in final merge stage");
return;
}
@@ -127,9 +125,11 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// get all the merged files
List<File> files = new ArrayList<File>(tempFileLocation.length);
- for (String tempLoc : tempFileLocation) {
+ for (String tempLoc : tempFileLocation)
+ {
File[] subFiles = new File(tempLoc).listFiles(fileFilter);
- if (null != subFiles && subFiles.length > 0) {
+ if (null != subFiles && subFiles.length > 0)
+ {
files.addAll(Arrays.asList(subFiles));
}
}
@@ -226,14 +226,13 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
}
/**
- * This method will be used to get the sorted sort temp row from the sort temp files
+ * This method will be used to get the sorted row
*
* @return sorted row
* @throws CarbonSortKeyAndGroupByException
*/
public Object[] next() {
- IntermediateSortTempRow sortTempRow = getSortedRecordFromFile();
- return sortStepRowHandler.convertIntermediateSortTempRowTo3Parted(sortTempRow);
+ return getSortedRecordFromFile();
}
/**
@@ -242,8 +241,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
* @return sorted record sorted record
* @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
- IntermediateSortTempRow row = null;
+ private Object[] getSortedRecordFromFile() throws CarbonDataWriterException {
+ Object[] row = null;
// poll the top object from heap
// heap maintains binary tree which is based on heap condition that will
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
index c7efbd9..57a19bd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.sort.sortdata;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.ExecutorService;
@@ -32,10 +32,12 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -67,8 +69,7 @@ public class SortDataRows {
private Semaphore semaphore;
private SortParameters parameters;
- private SortStepRowHandler sortStepRowHandler;
- private ThreadLocal<ByteBuffer> rowBuffer;
+
private int sortBufferSize;
private SortIntermediateFileMerger intermediateFileMerger;
@@ -78,7 +79,7 @@ public class SortDataRows {
public SortDataRows(SortParameters parameters,
SortIntermediateFileMerger intermediateFileMerger) {
this.parameters = parameters;
- this.sortStepRowHandler = new SortStepRowHandler(parameters);
+
this.intermediateFileMerger = intermediateFileMerger;
int batchSize = CarbonProperties.getInstance().getBatchSize();
@@ -86,12 +87,6 @@ public class SortDataRows {
this.sortBufferSize = Math.max(parameters.getSortBufferSize(), batchSize);
// observer of writing file in thread
this.threadStatusObserver = new ThreadStatusObserver();
- this.rowBuffer = new ThreadLocal<ByteBuffer>() {
- @Override protected ByteBuffer initialValue() {
- byte[] backedArray = new byte[2 * 1024 * 1024];
- return ByteBuffer.wrap(backedArray);
- }
- };
}
/**
@@ -135,7 +130,8 @@ public class SortDataRows {
semaphore.acquire();
dataSorterAndWriterExecutorService.execute(new DataSorterAndWriter(recordHolderListLocal));
} catch (InterruptedException e) {
- LOGGER.error(e, "exception occurred while trying to acquire a semaphore lock: ");
+ LOGGER.error(e,
+ "exception occurred while trying to acquire a semaphore lock: ");
throw new CarbonSortKeyAndGroupByException(e);
}
// create the new holder Array
@@ -162,7 +158,7 @@ public class SortDataRows {
}
intermediateFileMerger.startMergingIfPossible();
Object[][] recordHolderListLocal = recordHolderList;
- sizeLeft = sortBufferSize - entryCount;
+ sizeLeft = sortBufferSize - entryCount ;
if (sizeLeft > 0) {
System.arraycopy(rowBatch, 0, recordHolderListLocal, entryCount, sizeLeft);
}
@@ -216,6 +212,7 @@ public class SortDataRows {
locationChosen + File.separator + parameters.getTableName() +
System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
writeDataToFile(recordHolderList, this.entryCount, file);
+
}
startFileBasedMerge();
@@ -223,7 +220,7 @@ public class SortDataRows {
}
/**
- * Below method will be used to write data to sort temp file
+ * Below method will be used to write data to file
*
* @throws CarbonSortKeyAndGroupByException problem while writing
*/
@@ -236,9 +233,60 @@ public class SortDataRows {
parameters.getFileWriteBufferSize(), parameters.getSortTempCompressorName());
// write number of entries to the file
stream.writeInt(entryCountLocal);
+ int complexDimColCount = parameters.getComplexDimColCount();
+ int dimColCount = parameters.getDimColCount() + complexDimColCount;
+ DataType[] type = parameters.getMeasureDataType();
+ boolean[] noDictionaryDimnesionMapping = parameters.getNoDictionaryDimnesionColumn();
+ Object[] row = null;
for (int i = 0; i < entryCountLocal; i++) {
- sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToOutputStream(
- recordHolderList[i], stream, rowBuffer.get());
+ // get row from record holder list
+ row = recordHolderList[i];
+ int dimCount = 0;
+ // write dictionary and non dictionary dimensions here.
+ for (; dimCount < noDictionaryDimnesionMapping.length; dimCount++) {
+ if (noDictionaryDimnesionMapping[dimCount]) {
+ byte[] col = (byte[]) row[dimCount];
+ stream.writeShort(col.length);
+ stream.write(col);
+ } else {
+ stream.writeInt((int)row[dimCount]);
+ }
+ }
+ // write complex dimensions here.
+ for (; dimCount < dimColCount; dimCount++) {
+ byte[] value = (byte[])row[dimCount];
+ stream.writeShort(value.length);
+ stream.write(value);
+ }
+ // as measures are stored in separate array.
+ for (int mesCount = 0;
+ mesCount < parameters.getMeasureColCount(); mesCount++) {
+ Object value = row[mesCount + dimColCount];
+ if (null != value) {
+ stream.write((byte) 1);
+ DataType dataType = type[mesCount];
+ if (dataType == DataTypes.BOOLEAN) {
+ stream.writeBoolean((boolean) value);
+ } else if (dataType == DataTypes.SHORT) {
+ stream.writeShort((Short) value);
+ } else if (dataType == DataTypes.INT) {
+ stream.writeInt((Integer) value);
+ } else if (dataType == DataTypes.LONG) {
+ stream.writeLong((Long) value);
+ } else if (dataType == DataTypes.DOUBLE) {
+ stream.writeDouble((Double) value);
+ } else if (DataTypes.isDecimal(dataType)) {
+ BigDecimal val = (BigDecimal) value;
+ byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
+ stream.writeInt(bigDecimalInBytes.length);
+ stream.write(bigDecimalInBytes);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + type[mesCount]);
+ }
+ } else {
+ stream.write((byte) 0);
+ }
+ }
}
} catch (IOException e) {
throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
@@ -253,7 +301,7 @@ public class SortDataRows {
*
* @throws CarbonSortKeyAndGroupByException
*/
- private void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
+ public void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
CarbonDataProcessorUtil.deleteSortLocationIfExists(parameters.getTempFileLocation());
}
@@ -332,8 +380,7 @@ public class SortDataRows {
// intermediate merging of sort temp files will be triggered
intermediateFileMerger.addFileToMerge(sortTempFile);
LOGGER.info("Time taken to sort and write sort temp file " + sortTempFile + " is: " + (
- System.currentTimeMillis() - startTime) + ", sort temp file size in MB is "
- + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
+ System.currentTimeMillis() - startTime));
} catch (Throwable e) {
try {
threadStatusObserver.notifyFailed(e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
index 7e221a7..d726539 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java
@@ -21,7 +21,6 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.Comparator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -31,11 +30,14 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonThreadFactory;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.NonDictionaryUtil;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHolder> {
@@ -69,13 +71,20 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* return row
*/
- private IntermediateSortTempRow returnRow;
+ private Object[] returnRow;
+ private int dimCnt;
+ private int noDictDimCnt;
+ private int complexCnt;
+ private int measureCnt;
+ private boolean[] isNoDictionaryDimensionColumn;
+ private boolean[] isNoDictionarySortColumn;
+ private DataType[] measureDataTypes;
private int readBufferSize;
private String compressorName;
- private IntermediateSortTempRow[] currentBuffer;
+ private Object[][] currentBuffer;
- private IntermediateSortTempRow[] backupBuffer;
+ private Object[][] backupBuffer;
private boolean isBackupFilled;
@@ -95,9 +104,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
* totalRecordFetch
*/
private int totalRecordFetch;
- private TableFieldStat tableFieldStat;
- private SortStepRowHandler sortStepRowHandler;
- private Comparator<IntermediateSortTempRow> comparator;
+
/**
* Constructor to initialize
*
@@ -108,12 +115,16 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
public SortTempFileChunkHolder(File tempFile, SortParameters sortParameters, String tableName) {
// set temp file
this.tempFile = tempFile;
+ this.dimCnt = sortParameters.getDimColCount();
+ this.noDictDimCnt = sortParameters.getNoDictionaryCount();
+ this.complexCnt = sortParameters.getComplexDimColCount();
+ this.measureCnt = sortParameters.getMeasureColCount();
+ this.isNoDictionaryDimensionColumn = sortParameters.getNoDictionaryDimnesionColumn();
+ this.isNoDictionarySortColumn = sortParameters.getNoDictionarySortColumn();
+ this.measureDataTypes = sortParameters.getMeasureDataType();
this.readBufferSize = sortParameters.getBufferSize();
this.compressorName = sortParameters.getSortTempCompressorName();
- this.tableFieldStat = new TableFieldStat(sortParameters);
- this.sortStepRowHandler = new SortStepRowHandler(tableFieldStat);
- this.comparator = new IntermediateSortTempRowComparator(
- tableFieldStat.getIsSortColNoDictFlags());
+
this.executorService = Executors
.newFixedThreadPool(1, new CarbonThreadFactory("SafeSortTempChunkHolderPool:" + tableName));
}
@@ -167,12 +178,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
if (prefetch) {
fillDataForPrefetch();
} else {
- try {
- this.returnRow = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
- this.numberOfObjectRead++;
- } catch (IOException e) {
- throw new CarbonSortKeyAndGroupByException("Problem while reading rows", e);
- }
+ this.returnRow = getRowFromStream();
}
}
@@ -206,28 +212,86 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
/**
- * Read a batch of row from stream
- *
+ * Reads row from file
* @return Object[]
- * @throws IOException if error occurs while reading from stream
+ * @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow[] readBatchedRowFromStream(int expected) throws IOException {
- IntermediateSortTempRow[] holders = new IntermediateSortTempRow[expected];
- for (int i = 0; i < expected; i++) {
- IntermediateSortTempRow holder
- = sortStepRowHandler.readIntermediateSortTempRowFromInputStream(stream);
- holders[i] = holder;
+ private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
+ // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+
+ Object[] holder = new Object[3];
+ int index = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[dimCnt - noDictDimCnt];
+ byte[][] nonDicArray = new byte[noDictDimCnt + complexCnt][];
+ Object[] measures = new Object[measureCnt];
+ try {
+ // read dimension values
+ for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
+ if (isNoDictionaryDimensionColumn[i]) {
+ short len = stream.readShort();
+ byte[] array = new byte[len];
+ stream.readFully(array);
+ nonDicArray[nonDicIndex++] = array;
+ } else {
+ dim[index++] = stream.readInt();
+ }
+ }
+
+ for (int i = 0; i < complexCnt; i++) {
+ short len = stream.readShort();
+ byte[] array = new byte[len];
+ stream.readFully(array);
+ nonDicArray[nonDicIndex++] = array;
+ }
+
+ index = 0;
+ // read measure values
+ for (int i = 0; i < measureCnt; i++) {
+ if (stream.readByte() == 1) {
+ DataType dataType = measureDataTypes[i];
+ if (dataType == DataTypes.BOOLEAN) {
+ measures[index++] = stream.readBoolean();
+ } else if (dataType == DataTypes.SHORT) {
+ measures[index++] = stream.readShort();
+ } else if (dataType == DataTypes.INT) {
+ measures[index++] = stream.readInt();
+ } else if (dataType == DataTypes.LONG) {
+ measures[index++] = stream.readLong();
+ } else if (dataType == DataTypes.DOUBLE) {
+ measures[index++] = stream.readDouble();
+ } else if (DataTypes.isDecimal(dataType)) {
+ int len = stream.readInt();
+ byte[] buff = new byte[len];
+ stream.readFully(buff);
+ measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
+ } else {
+ throw new IllegalArgumentException("unsupported data type:" + dataType);
+ }
+ } else {
+ measures[index++] = null;
+ }
+ }
+
+ NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
+
+ // increment number if record read
+ this.numberOfObjectRead++;
+ } catch (IOException e) {
+ LOGGER.error("Problme while reading the madkey fom sort temp file");
+ throw new CarbonSortKeyAndGroupByException("Problem while reading the sort temp file ", e);
}
- this.numberOfObjectRead += expected;
- return holders;
+
+ //return out row
+ return holder;
}
/**
- * below method will be used to get the sort temp row
+ * below method will be used to get the row
*
* @return row
*/
- public IntermediateSortTempRow getRow() {
+ public Object[] getRow() {
return this.returnRow;
}
@@ -266,7 +330,31 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
}
@Override public int compareTo(SortTempFileChunkHolder other) {
- return comparator.compare(returnRow, other.getRow());
+ int diff = 0;
+ int index = 0;
+ int noDictionaryIndex = 0;
+ int[] leftMdkArray = (int[]) returnRow[0];
+ int[] rightMdkArray = (int[]) other.returnRow[0];
+ byte[][] leftNonDictArray = (byte[][]) returnRow[1];
+ byte[][] rightNonDictArray = (byte[][]) other.returnRow[1];
+ for (boolean isNoDictionary : isNoDictionarySortColumn) {
+ if (isNoDictionary) {
+ diff = UnsafeComparer.INSTANCE
+ .compareTo(leftNonDictArray[noDictionaryIndex], rightNonDictArray[noDictionaryIndex]);
+ if (diff != 0) {
+ return diff;
+ }
+ noDictionaryIndex++;
+ } else {
+ diff = leftMdkArray[index] - rightMdkArray[index];
+ if (diff != 0) {
+ return diff;
+ }
+ index++;
+ }
+
+ }
+ return diff;
}
@Override public boolean equals(Object obj) {
@@ -284,7 +372,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
@Override public int hashCode() {
int hash = 0;
- hash += tableFieldStat.hashCode();
+ hash += 31 * measureCnt;
+ hash += 31 * dimCnt;
+ hash += 31 * complexCnt;
hash += tempFile.hashCode();
return hash;
}
@@ -324,12 +414,16 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
/**
* This method will read the records from sort temp file and keep it in a buffer
*
- * @param numberOfRecords number of records to be read
- * @return batch of intermediate sort temp row
- * @throws IOException if error occurs while reading reading records
+ * @param numberOfRecords
+ * @return
+ * @throws CarbonSortKeyAndGroupByException
*/
- private IntermediateSortTempRow[] prefetchRecordsFromFile(int numberOfRecords)
- throws IOException {
- return readBatchedRowFromStream(numberOfRecords);
+ private Object[][] prefetchRecordsFromFile(int numberOfRecords)
+ throws CarbonSortKeyAndGroupByException {
+ Object[][] records = new Object[numberOfRecords][];
+ for (int i = 0; i < numberOfRecords; i++) {
+ records[i] = getRowFromStream();
+ }
+ return records;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
deleted file mode 100644
index 0d1303a..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/TableFieldStat.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.sort.sortdata;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * This class is used to hold field information for a table during data loading. These information
- * will be used to convert/construct/destruct row in sort process step. Because complex field is
- * processed the same as no-dict-no-sort-simple-dimension, so we treat them as the same and use
- * `no-dict-no-sort-dim` related variable to represent them in this class.
- */
-public class TableFieldStat implements Serializable {
- private static final long serialVersionUID = 201712070950L;
- private int dictSortDimCnt = 0;
- private int dictNoSortDimCnt = 0;
- private int noDictSortDimCnt = 0;
- private int noDictNoSortDimCnt = 0;
- // whether sort column is of dictionary type or not
- private boolean[] isSortColNoDictFlags;
- private int measureCnt;
- private DataType[] measureDataType;
-
- // indices for dict & sort dimension columns
- private int[] dictSortDimIdx;
- // indices for dict & no-sort dimension columns
- private int[] dictNoSortDimIdx;
- // indices for no-dict & sort dimension columns
- private int[] noDictSortDimIdx;
- // indices for no-dict & no-sort dimension columns, including complex columns
- private int[] noDictNoSortDimIdx;
- // indices for measure columns
- private int[] measureIdx;
-
- public TableFieldStat(SortParameters sortParameters) {
- int noDictDimCnt = sortParameters.getNoDictionaryCount();
- int complexDimCnt = sortParameters.getComplexDimColCount();
- int dictDimCnt = sortParameters.getDimColCount() - noDictDimCnt;
- this.isSortColNoDictFlags = sortParameters.getNoDictionarySortColumn();
- int sortColCnt = isSortColNoDictFlags.length;
- for (boolean flag : isSortColNoDictFlags) {
- if (flag) {
- noDictSortDimCnt++;
- } else {
- dictSortDimCnt++;
- }
- }
- this.measureCnt = sortParameters.getMeasureColCount();
- this.measureDataType = sortParameters.getMeasureDataType();
-
- // be careful that the default value is 0
- this.dictSortDimIdx = new int[dictSortDimCnt];
- this.dictNoSortDimIdx = new int[dictDimCnt - dictSortDimCnt];
- this.noDictSortDimIdx = new int[noDictSortDimCnt];
- this.noDictNoSortDimIdx = new int[noDictDimCnt + complexDimCnt - noDictSortDimCnt];
- this.measureIdx = new int[measureCnt];
-
- int tmpNoDictSortCnt = 0;
- int tmpNoDictNoSortCnt = 0;
- int tmpDictSortCnt = 0;
- int tmpDictNoSortCnt = 0;
- boolean[] isDimNoDictFlags = sortParameters.getNoDictionaryDimnesionColumn();
-
- for (int i = 0; i < isDimNoDictFlags.length; i++) {
- if (isDimNoDictFlags[i]) {
- if (i < sortColCnt && isSortColNoDictFlags[i]) {
- noDictSortDimIdx[tmpNoDictSortCnt++] = i;
- } else {
- noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = i;
- }
- } else {
- if (i < sortColCnt && !isSortColNoDictFlags[i]) {
- dictSortDimIdx[tmpDictSortCnt++] = i;
- } else {
- dictNoSortDimIdx[tmpDictNoSortCnt++] = i;
- }
- }
- }
- dictNoSortDimCnt = tmpDictNoSortCnt;
-
- int base = isDimNoDictFlags.length;
- // adding complex dimension columns
- for (int i = 0; i < complexDimCnt; i++) {
- noDictNoSortDimIdx[tmpNoDictNoSortCnt++] = base + i;
- }
- noDictNoSortDimCnt = tmpNoDictNoSortCnt;
-
- base += complexDimCnt;
- // indices for measure columns
- for (int i = 0; i < measureCnt; i++) {
- measureIdx[i] = base + i;
- }
- }
-
- public int getDictSortDimCnt() {
- return dictSortDimCnt;
- }
-
- public int getDictNoSortDimCnt() {
- return dictNoSortDimCnt;
- }
-
- public int getNoDictSortDimCnt() {
- return noDictSortDimCnt;
- }
-
- public int getNoDictNoSortDimCnt() {
- return noDictNoSortDimCnt;
- }
-
- public boolean[] getIsSortColNoDictFlags() {
- return isSortColNoDictFlags;
- }
-
- public int getMeasureCnt() {
- return measureCnt;
- }
-
- public DataType[] getMeasureDataType() {
- return measureDataType;
- }
-
- public int[] getDictSortDimIdx() {
- return dictSortDimIdx;
- }
-
- public int[] getDictNoSortDimIdx() {
- return dictNoSortDimIdx;
- }
-
- public int[] getNoDictSortDimIdx() {
- return noDictSortDimIdx;
- }
-
- public int[] getNoDictNoSortDimIdx() {
- return noDictNoSortDimIdx;
- }
-
- public int[] getMeasureIdx() {
- return measureIdx;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof TableFieldStat)) return false;
- TableFieldStat that = (TableFieldStat) o;
- return dictSortDimCnt == that.dictSortDimCnt
- && dictNoSortDimCnt == that.dictNoSortDimCnt
- && noDictSortDimCnt == that.noDictSortDimCnt
- && noDictNoSortDimCnt == that.noDictNoSortDimCnt
- && measureCnt == that.measureCnt;
- }
-
- @Override public int hashCode() {
- return Objects.hash(dictSortDimCnt, dictNoSortDimCnt, noDictSortDimCnt,
- noDictNoSortDimCnt, measureCnt);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 64e50b0..efd715c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -393,8 +392,7 @@ public final class CarbonDataProcessorUtil {
*
* @return data directory path
*/
- public static String createCarbonStoreLocation(String factStoreLocation,
- String databaseName, String tableName, String segmentId) {
+ public static String createCarbonStoreLocation(String databaseName, String tableName, String segmentId) {
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName);
return CarbonTablePath.getSegmentPath(carbonTable.getTablePath(), segmentId);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 3302094..70c26ab 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -433,15 +433,6 @@ public final class CarbonLoaderUtil {
}
- public static String readCurrentTime() {
- SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- String date = null;
-
- date = sdf.format(new Date());
-
- return date;
- }
-
public static boolean isValidEscapeSequence(String escapeChar) {
return escapeChar.equalsIgnoreCase(NEW_LINE.getName()) ||
escapeChar.equalsIgnoreCase(CARRIAGE_RETURN.getName()) ||
[08/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 9391ebd..44f7c07 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -16,20 +16,10 @@
*/
package org.apache.carbondata.core.scan.filter.executer;
-import java.util.ArrayList;
import java.util.BitSet;
-import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
@@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil;
public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
* @param dimColResolvedFilterInfo
* @param segmentProperties
*/
@@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
}
/**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.flip(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
* Check if scan is required on given block based on min and max value
*/
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
@@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
return bitSet;
}
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index fad37fc..fe983a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -40,13 +40,13 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
- protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ private DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
private SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -82,48 +82,50 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- DimensionColumnDataChunk[] dimensionColumnDataChunks =
- dimensionRawColumnChunk.convertToDimColDataChunks();
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ DimensionColumnPage[] dimensionColumnPages =
+ dimensionRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
- for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
- BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i],
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
ColumnPage[] ColumnPages =
- measureRawColumnChunk.convertToColumnPage();
+ measureRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < ColumnPages.length; i++) {
BitSet bitSet =
getFilteredIndexesForMeasure(
- measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i],
useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(),
+ rawBlockletColumnChunks.getBitSetGroup(),
i,
msrType);
bitSetGroup.setBitSet(bitSet, i);
@@ -133,7 +135,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return null;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
@@ -275,43 +278,41 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether applying filtered based on previous bitset will be optimal
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getExcludeFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
- numberOfRows);
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter based on previous filtered bitset
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
- * @param numberOfRows
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
- BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
+ BitSetGroup prvBitSetGroup, int pageNumber) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
@@ -319,11 +320,11 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -336,8 +337,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -351,7 +352,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
int startIndex = 0;
@@ -361,10 +362,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.flip(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.flip(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -373,7 +374,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
@@ -390,7 +391,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.flip(j);
@@ -403,14 +404,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.flip(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.flip(j);
}
}
@@ -419,26 +420,30 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 53d3068..a64341e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public interface FilterExecuter {
@@ -32,8 +32,8 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException;
+ BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException;
/**
* apply range filter on a row
@@ -54,7 +54,7 @@ public interface FilterExecuter {
/**
* It just reads necessary block for filter executor, it does not uncompress the data.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
*/
- void readBlocks(BlocksChunkHolder blockChunkHolder)throws IOException;
+ void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index 9c4c7ba..b683fd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -23,7 +23,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -41,26 +41,31 @@ public class ImplicitIncludeFilterExecutorImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeline)
- throws FilterUnsupportedException {
- BitSetGroup bitSetGroup = new BitSetGroup(blockChunkHolder.getDataBlock().numberOfPages());
- for (int i = 0; i < blockChunkHolder.getDataBlock().numberOfPages(); i++) {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeline) {
+ BitSetGroup bitSetGroup = new BitSetGroup(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ for (int i = 0; i < rawBlockletColumnChunks.getDataBlock().numberOfPages(); i++) {
bitSetGroup.setBitSet(
- setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().getPageRowCount(i)), i);
+ setBitSetForCompleteDimensionData(
+ rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)), i);
}
return bitSetGroup;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return null;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 1cbc0bc..e4da26f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -24,14 +24,14 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
@@ -59,7 +59,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
/**
* It fills BitSet with row index which matches filter key
*/
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
@@ -70,7 +70,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
byte[] filterVal = filterValues[i];
for (int rowId = 0; rowId < numerOfRows; rowId++) {
byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo);
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
bitSet.set(rowId);
}
@@ -85,20 +85,21 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -133,10 +134,10 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
BitSet bitSet = new BitSet(1);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
+ byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex);
+ byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex);
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index fe1421c..05328f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -41,12 +41,12 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class IncludeFilterExecuterImpl implements FilterExecuter {
protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
protected SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -86,44 +86,46 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) {
@@ -132,16 +134,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
}
@@ -293,43 +295,43 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether previous indexes can be optimal to apply filter on dimension column
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber,
numberOfRows);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter on dimension
* column based on previous filtered indexes
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
* @param numberOfRows
* @return filtered bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet(numberOfRows);
@@ -337,11 +339,11 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult == 0) {
bitSet.set(index);
}
@@ -350,8 +352,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult == 0) {
bitSet.set(index);
}
@@ -360,7 +362,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int startIndex = 0;
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -369,10 +371,10 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -381,7 +383,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -394,7 +396,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.set(j);
@@ -407,14 +409,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.set(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.set(j);
}
}
@@ -423,26 +425,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
+ @Override
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = null;
int columnIndex = 0;
- int blockIndex = 0;
+ int chunkIndex = 0;
boolean isScanRequired = false;
if (isDimensionPresentInCurrentBlock) {
filterValues = dimColumnExecuterInfo.getFilterKeys();
columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- isScanRequired =
- isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues);
-
+ chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues);
} else if (isMeasurePresentInCurrentBlock) {
columnIndex = msrColumnEvaluatorInfo.getColumnIndex();
- blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties
- .getLastDimensionColOrdinal();
- isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex],
+ chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(columnIndex) +
+ segmentProperties.getLastDimensionColOrdinal();
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType());
}
@@ -492,20 +493,23 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return false;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- } else if (isMeasurePresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ } else if (isMeasurePresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 87273bb..9f0afb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class OrFilterExecuterImpl implements FilterExecuter {
@@ -35,30 +35,33 @@ public class OrFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false);
- BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, false);
+ BitSetGroup rightFilters = rightExecuter.applyFilter(rawBlockletColumnChunks, false);
leftFilters.or(rightFilters);
- blockChunkHolder.setBitSetGroup(leftFilters);
+ rawBlockletColumnChunks.setBitSetGroup(leftFilters);
return leftFilters;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return leftExecuter.applyFilter(value, dimOrdinalMax) ||
rightExecuter.applyFilter(value, dimOrdinalMax);
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
leftFilters.or(rightFilters);
return leftFilters;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- leftExecuter.readBlocks(blockChunkHolder);
- rightExecuter.readBlocks(blockChunkHolder);
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ leftExecuter.readColumnChunks(rawBlockletColumnChunks);
+ rightExecuter.readColumnChunks(rawBlockletColumnChunks);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 797fe9d..34555e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -22,11 +22,10 @@ import java.util.BitSet;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -39,8 +38,7 @@ import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedExc
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -60,25 +58,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* it has index at which given dimension is stored in file
*/
- protected int dimensionBlocksIndex;
+ private int dimensionChunkIndex;
/**
* flag to check whether the filter dimension is present in current block list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean isDimensionPresentInCurrentBlock;
- boolean lessThanExp;
- boolean lessThanEqualExp;
- boolean greaterThanExp;
- boolean greaterThanEqualExp;
- boolean startBlockMinIsDefaultStart;
- boolean endBlockMaxisDefaultEnd;
- boolean isRangeFullyCoverBlock;
+ private boolean isDimensionPresentInCurrentBlock;
+ private boolean lessThanExp;
+ private boolean lessThanEqualExp;
+ private boolean greaterThanExp;
+ private boolean greaterThanEqualExp;
+ private boolean startBlockMinIsDefaultStart;
+ private boolean endBlockMaxisDefaultEnd;
+ private boolean isRangeFullyCoverBlock;
public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- MeasureColumnResolvedFilterInfo msrColEvaluatorInfo, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
+ Expression exp, byte[][] filterRangeValues, SegmentProperties segmentProperties) {
this.dimColEvaluatorInfo = dimColEvaluatorInfo;
this.exp = exp;
@@ -91,7 +87,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startBlockMinIsDefaultStart = false;
endBlockMaxisDefaultEnd = false;
isRangeFullyCoverBlock = false;
- initDimensionBlockIndexes();
+ initDimensionChunkIndexes();
ifDefaultValueMatchesFilter();
}
@@ -100,13 +96,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* This method will initialize the dimension info for the current block to be
* used for filtering the data
*/
- private void initDimensionBlockIndexes() {
+ private void initDimensionChunkIndexes() {
// find the dimension in the current block dimensions list
CarbonDimension dimensionFromCurrentBlock =
segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension());
if (null != dimensionFromCurrentBlock) {
dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- this.dimensionBlocksIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ this.dimensionChunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimensionFromCurrentBlock.getOrdinal());
isDimensionPresentInCurrentBlock = true;
}
@@ -136,14 +132,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to apply the filter.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @return
* @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- return applyNoAndDirectFilter(blockChunkHolder);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ return applyNoAndDirectFilter(rawBlockletColumnChunks);
}
/**
@@ -272,32 +268,32 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Filter Min <-----------------------------------------------> Filter Max
if (isDimensionPresentInCurrentBlock) {
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || (
- (greaterThanExp == true) && (
+ (greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) {
// completely out of block boundary
isScanRequired = false;
} else {
- if (((greaterThanExp == true) && (
+ if (((greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) {
startBlockMinIsDefaultStart = true;
}
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) {
endBlockMaxisDefaultEnd = true;
}
- if (startBlockMinIsDefaultStart == true && endBlockMaxisDefaultEnd == true) {
+ if (startBlockMinIsDefaultStart && endBlockMaxisDefaultEnd) {
isRangeFullyCoverBlock = true;
}
}
@@ -313,7 +309,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* @param blockMinValue
* @return
*/
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangesValues;
int columnIndex = this.dimColEvaluatorInfo.getColumnIndex();
@@ -329,50 +326,49 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* Method to apply the Range Filter.
* @param blockChunkHolder
* @return
- * @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
+ private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder)
+ throws IOException {
// In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be
// false, in that scenario the default values of the column should be shown.
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, true);
+ int numberOfRows = blockChunkHolder.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true);
}
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex] =
+ blockChunkHolder.getDataBlock().readDimensionChunk(
+ blockChunkHolder.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i],
this.filterRangesValues)) {
- if (isRangeFullyCoverBlock == true) {
+ if (isRangeFullyCoverBlock) {
// Set all the bits in this case as filter Min Max values cover the whole block.
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -380,12 +376,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
return bitSetGroup;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
/**
@@ -394,12 +390,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int startIndex = 0;
@@ -410,13 +406,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
// Get the Min Value
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -428,8 +424,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
- .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) {
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage
+ .getChunkData(dimensionColumnPage.getInvertedIndex(start)))) > 0) {
start = start + 1;
}
}
@@ -440,14 +436,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// Get the Max value
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -458,7 +454,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
if ((ByteUtil.compare(filterValues[1],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0)) {
start = start - 1;
}
@@ -469,18 +465,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
for (int j = startMin; j <= endMax; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
return bitSet;
}
- private void updateForNoDictionaryColumn(int start, int end, DimensionColumnDataChunk dataChunk,
+ private void updateForNoDictionaryColumn(int start, int end, DimensionColumnPage dataChunk,
BitSet bitset) {
for (int j = start; j <= end; j++) {
if (dataChunk.compareTo(j, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) == 0
@@ -496,31 +492,31 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* be called if the column is sorted default so column index
* mapping will be present for accesaing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
- // if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ // if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
byte[][] filterValues = this.filterRangesValues;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
+ if (dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int startMin = 0;
int endMax = 0;
int startIndex = 0;
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -532,7 +528,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage.getChunkData(start)))
> 0) {
start = start + 1;
}
@@ -542,14 +538,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startMin = startIndex;
}
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -559,7 +555,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
- if ((ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start))
+ if ((ByteUtil.compare(filterValues[1], dimensionColumnPage.getChunkData(start))
< 0)) {
start = start - 1;
}
@@ -575,8 +571,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
} else {
byte[] defaultValue = null;
@@ -585,7 +581,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex);
+ segmentProperties.getDimensions().get(dimensionChunkIndex);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -601,55 +597,37 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// evaluate result for lower range value first and then perform and operation in the
// upper range value in order to compute the final result
- bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[0],
+ bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[0],
numerOfRows);
BitSet upperRangeBitSet =
- evaluateLessThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[1],
+ evaluateLessThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[1],
numerOfRows);
bitSet.and(upperRangeBitSet);
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet, defaultValue);
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
}
/**
- * This method will compare the selected data against null values and
- * flip the bitSet if any null value is found
- *
- * @param dimensionColumnDataChunk
- * @param bitSet
- */
- private void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk, BitSet bitSet) {
- if (!bitSet.isEmpty()) {
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- if (dimensionColumnDataChunk.compareTo(i, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY)
- == 0) {
- bitSet.flip(i);
- }
- }
- }
- }
-
- /**
* This method will evaluate the result for filter column based on the lower range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateGreaterThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (greaterThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) > 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) > 0)) {
bitSet.set(i);
}
}
} else if (greaterThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) >= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) >= 0)) {
bitSet.set(i);
}
}
@@ -660,23 +638,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* This method will evaluate the result for filter column based on the upper range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateLessThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (lessThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) < 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) < 0)) {
bitSet.set(i);
}
}
} else if (lessThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) <= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) <= 0)) {
bitSet.set(i);
}
}
@@ -686,16 +664,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to read the blocks.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @throws IOException
*/
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 5707eb4..d7bec7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -34,7 +34,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -48,26 +48,29 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, !isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureExcludeFilterExecutorImpl on row");
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 8bcc53f..c874fc5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -33,7 +33,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -47,15 +47,16 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, isDefaultValuePresentInFilterValues);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages(),
+ numberOfRows, isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureIncludeFilterExecutorImpl on row");
}
@@ -66,7 +67,8 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}
[29/54] [abbrv] carbondata git commit: [CARBONDATA-1997] Add
CarbonWriter SDK API
Posted by ra...@apache.org.
[CARBONDATA-1997] Add CarbonWriter SDK API
Added a new module called store-sdk, and added a CarbonWriter API, it can be used to write Carbondata files to a specified folder, without Spark and Hadoop dependency. User can use this API in any environment.
This closes #1967
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1d827c7b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1d827c7b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1d827c7b
Branch: refs/heads/master
Commit: 1d827c7b515a21d936769424c282d2aafb1ef6b7
Parents: ce6e71c
Author: Jacky Li <ja...@qq.com>
Authored: Sat Feb 10 19:44:23 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../org/apache/carbondata/common/Strings.java | 40 ++++
.../apache/carbondata/common/StringsSuite.java | 53 +++++
.../core/metadata/schema/table/CarbonTable.java | 7 +
.../schema/table/CarbonTableBuilder.java | 72 +++++++
.../core/metadata/schema/table/TableSchema.java | 7 +
.../schema/table/TableSchemaBuilder.java | 107 ++++++++++
.../schema/table/CarbonTableBuilderSuite.java | 86 ++++++++
.../metadata/schema/table/CarbonTableTest.java | 12 +-
.../schema/table/TableSchemaBuilderSuite.java | 56 ++++++
.../carbondata/spark/util/DataLoadingUtil.scala | 45 +++++
pom.xml | 7 +
store/sdk/pom.xml | 130 +++++++++++++
.../carbondata/sdk/file/CSVCarbonWriter.java | 89 +++++++++
.../carbondata/sdk/file/CarbonWriter.java | 51 +++++
.../sdk/file/CarbonWriterBuilder.java | 194 +++++++++++++++++++
.../org/apache/carbondata/sdk/file/Field.java | 74 +++++++
.../org/apache/carbondata/sdk/file/Schema.java | 74 +++++++
.../sdk/file/CSVCarbonWriterSuite.java | 127 ++++++++++++
18 files changed, 1225 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/common/src/main/java/org/apache/carbondata/common/Strings.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/Strings.java b/common/src/main/java/org/apache/carbondata/common/Strings.java
new file mode 100644
index 0000000..23288dd
--- /dev/null
+++ b/common/src/main/java/org/apache/carbondata/common/Strings.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common;
+
+import java.util.Objects;
+
+public class Strings {
+
+ /**
+ * Provide same function as mkString in Scala.
+ * This is added to avoid JDK 8 dependency.
+ */
+ public static String mkString(String[] strings, String delimeter) {
+ Objects.requireNonNull(strings);
+ Objects.requireNonNull(delimeter);
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < strings.length; i++) {
+ builder.append(strings[i]);
+ if (i != strings.length - 1) {
+ builder.append(delimeter);
+ }
+ }
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/carbondata/common/StringsSuite.java b/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
new file mode 100644
index 0000000..65da32b
--- /dev/null
+++ b/common/src/test/java/org/apache/carbondata/common/StringsSuite.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StringsSuite {
+
+ @Test(expected = NullPointerException.class)
+ public void testMkStringNullString() {
+ Strings.mkString(null, ",");
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testMkStringNullDelimeter() {
+ Strings.mkString(new String[]{"abc"}, null);
+ }
+
+ @Test
+ public void testMkString() {
+ String[] strings = new String[]{};
+ String output = Strings.mkString(strings, ",");
+ Assert.assertTrue(output.length() == 0);
+
+ strings = new String[]{"abc"};
+ output = Strings.mkString(strings, ",");
+ Assert.assertEquals("abc", output);
+
+ strings = new String[]{"abc", "def"};
+ output = Strings.mkString(strings, ",");
+ Assert.assertEquals("abc,def", output);
+
+ strings = new String[]{"abc", "def", "ghj"};
+ output = Strings.mkString(strings, ",");
+ Assert.assertEquals("abc,def,ghj", output);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6ed1197..4d54ad5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -875,4 +875,11 @@ public class CarbonTable implements Serializable {
return projection;
}
+
+ /**
+ * Create a {@link CarbonTableBuilder} to create {@link CarbonTable}
+ */
+ public static CarbonTableBuilder builder() {
+ return new CarbonTableBuilder();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilder.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilder.java
new file mode 100644
index 0000000..27808f8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilder.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.util.ArrayList;
+import java.util.Objects;
+
+/**
+ * Builder for {@link CarbonTable}
+ */
+public class CarbonTableBuilder {
+
+ private String tableName;
+ private String databaseName;
+ private String tablePath;
+ private TableSchema tableSchema;
+
+ public CarbonTableBuilder tableName(String tableName) {
+ Objects.requireNonNull(tableName, "tableName should not be null");
+ this.tableName = tableName;
+ return this;
+ }
+
+ public CarbonTableBuilder databaseName(String databaseName) {
+ Objects.requireNonNull(databaseName, "databaseName should not be null");
+ this.databaseName = databaseName;
+ return this;
+ }
+
+ public CarbonTableBuilder tablePath(String tablePath) {
+ Objects.requireNonNull(tablePath, "tablePath should not be null");
+ this.tablePath = tablePath;
+ return this;
+ }
+
+ public CarbonTableBuilder tableSchema(TableSchema tableSchema) {
+ Objects.requireNonNull(tableSchema, "tableSchema should not be null");
+ this.tableSchema = tableSchema;
+ return this;
+ }
+
+ public CarbonTable build() {
+ Objects.requireNonNull(tableName, "tableName should not be null");
+ Objects.requireNonNull(databaseName, "databaseName should not be null");
+ Objects.requireNonNull(tablePath, "tablePath should not be null");
+ Objects.requireNonNull(tableSchema, "tableSchema should not be null");
+
+ TableInfo tableInfo = new TableInfo();
+ tableInfo.setDatabaseName(databaseName);
+ tableInfo.setTableUniqueName(databaseName + "_" + tableName);
+ tableInfo.setFactTable(tableSchema);
+ tableInfo.setTablePath(tablePath);
+ tableInfo.setLastUpdatedTime(System.currentTimeMillis());
+ tableInfo.setDataMapSchemaList(new ArrayList<DataMapSchema>(0));
+ return CarbonTable.buildFromTableInfo(tableInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 8fdfbab..fff1a74 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -294,4 +294,11 @@ public class TableSchema implements Serializable, Writable {
return dataMapSchema;
}
+ /**
+ * Create a {@link TableSchemaBuilder} to create {@link TableSchema}
+ */
+ public static TableSchemaBuilder builder() {
+ return new TableSchemaBuilder();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
new file mode 100644
index 0000000..88774ec
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilder.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
+import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+/**
+ * Builder for {@link TableSchema}
+ */
+public class TableSchemaBuilder {
+
+ private int ordinal = 0;
+
+ private List<ColumnSchema> sortColumns = new LinkedList<>();
+
+ private List<ColumnSchema> otherColumns = new LinkedList<>();
+
+ public TableSchema build() {
+ TableSchema schema = new TableSchema();
+ schema.setTableId(UUID.randomUUID().toString());
+ schema.setPartitionInfo(null);
+ schema.setBucketingInfo(null);
+ SchemaEvolution schemaEvol = new SchemaEvolution();
+ schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
+ schema.setSchemaEvalution(schemaEvol);
+ List<ColumnSchema> allColumns = new LinkedList<>(sortColumns);
+ allColumns.addAll(otherColumns);
+ schema.setListOfColumns(allColumns);
+ return schema;
+ }
+
+ public TableSchemaBuilder addColumn(StructField field, boolean isSortColumn) {
+ Objects.requireNonNull(field);
+ checkRepeatColumnName(field);
+ ColumnSchema newColumn = new ColumnSchema();
+ newColumn.setColumnName(field.getFieldName());
+ newColumn.setDataType(field.getDataType());
+ newColumn.setDimensionColumn(isSortColumn || field.getDataType() == DataTypes.STRING);
+ newColumn.setSchemaOrdinal(ordinal++);
+ newColumn.setColumnar(true);
+ newColumn.setColumnUniqueId(UUID.randomUUID().toString());
+ newColumn.setColumnReferenceId(newColumn.getColumnUniqueId());
+ newColumn.setEncodingList(createEncoding(field.getDataType(), isSortColumn));
+
+ if (isSortColumn) {
+ sortColumns.add(newColumn);
+ } else {
+ otherColumns.add(newColumn);
+ }
+ return this;
+ }
+
+ /**
+ * Throw exception if {@param field} name is repeated
+ */
+ private void checkRepeatColumnName(StructField field) {
+ for (ColumnSchema column : sortColumns) {
+ if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) {
+ throw new IllegalArgumentException("column name already exists");
+ }
+ }
+ for (ColumnSchema column : otherColumns) {
+ if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) {
+ throw new IllegalArgumentException("column name already exists");
+ }
+ }
+ }
+
+ private List<Encoding> createEncoding(DataType dataType, boolean isSortColumn) {
+ List<Encoding> encodings = new LinkedList<>();
+ if (dataType == DataTypes.TIMESTAMP || dataType == DataTypes.DATE) {
+ encodings.add(Encoding.DIRECT_DICTIONARY);
+ }
+ if (isSortColumn) {
+ encodings.add(Encoding.INVERTED_INDEX);
+ }
+ return encodings;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilderSuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilderSuite.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilderSuite.java
new file mode 100644
index 0000000..83b65a0
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableBuilderSuite.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class CarbonTableBuilderSuite {
+
+ TableSchema schema = CarbonTableTest.getTableSchema("t1");
+
+ @Test(expected = NullPointerException.class)
+ public void testNullTableName() {
+ TableSchema schema = CarbonTableTest.getTableSchema(null);
+ CarbonTable table = CarbonTable.builder()
+ .tableName(null)
+ .databaseName("db1")
+ .tableSchema(schema)
+ .tablePath("_temp")
+ .build();
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testNullDbName() {
+ CarbonTable table = CarbonTable.builder()
+ .tableName(schema.getTableName())
+ .databaseName(null)
+ .tableSchema(schema)
+ .tablePath("_temp")
+ .build();
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testNullSchema() {
+ CarbonTable table = CarbonTable.builder()
+ .tableName(schema.getTableName())
+ .databaseName("db1")
+ .tableSchema(null)
+ .tablePath("_temp")
+ .build();
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testNullTablePath() {
+ CarbonTable table = CarbonTable.builder()
+ .tableName(schema.getTableName())
+ .databaseName("db1")
+ .tableSchema(schema)
+ .tablePath(null)
+ .build();
+ }
+
+ @Test
+ public void testBuilder() {
+ CarbonTable table = CarbonTable.builder()
+ .tableName(schema.getTableName())
+ .databaseName("db1")
+ .tableSchema(schema)
+ .tablePath("_temp")
+ .build();
+ Assert.assertEquals(schema.getTableName(), table.getTableName());
+ Assert.assertEquals("db1", table.getDatabaseName());
+ Assert.assertEquals("_temp", table.getTablePath());
+ Assert.assertEquals(schema.getTableName(), table.getAbsoluteTableIdentifier().getTableName());
+ Assert.assertEquals("db1", table.getAbsoluteTableIdentifier().getDatabaseName());
+ Assert.assertEquals("_temp", table.getAbsoluteTableIdentifier().getTablePath());
+ Assert.assertEquals("db1_t1", table.getTableUniqueName());
+ Assert.assertEquals(schema, table.getTableInfo().getFactTable());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
index a47b7fd..0266df4 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
@@ -68,7 +68,7 @@ public class CarbonTableTest extends TestCase {
assertTrue(carbonTable.getDimensionByName("carbonTestTable", "IMEI").equals(dimension));
}
- private ColumnSchema getColumnarDimensionColumn() {
+ static ColumnSchema getColumnarDimensionColumn() {
ColumnSchema dimColumn = new ColumnSchema();
dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI");
@@ -83,7 +83,7 @@ public class CarbonTableTest extends TestCase {
return dimColumn;
}
- private ColumnSchema getColumnarMeasureColumn() {
+ static ColumnSchema getColumnarMeasureColumn() {
ColumnSchema dimColumn = new ColumnSchema();
dimColumn.setColumnName("IMEI_COUNT");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
@@ -91,23 +91,23 @@ public class CarbonTableTest extends TestCase {
return dimColumn;
}
- private TableSchema getTableSchema() {
+ static TableSchema getTableSchema(String tableName) {
TableSchema tableSchema = new TableSchema();
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
columnSchemaList.add(getColumnarDimensionColumn());
columnSchemaList.add(getColumnarMeasureColumn());
tableSchema.setListOfColumns(columnSchemaList);
tableSchema.setTableId(UUID.randomUUID().toString());
- tableSchema.setTableName("carbonTestTable");
+ tableSchema.setTableName(tableName);
return tableSchema;
}
- private TableInfo getTableInfo(long timeStamp) {
+ static private TableInfo getTableInfo(long timeStamp) {
TableInfo info = new TableInfo();
info.setDatabaseName("carbonTestDatabase");
info.setLastUpdatedTime(timeStamp);
info.setTableUniqueName("carbonTestDatabase_carbonTestTable");
- info.setFactTable(getTableSchema());
+ info.setFactTable(getTableSchema("carbonTestTable"));
info.setTablePath("testore");
return info;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilderSuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilderSuite.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilderSuite.java
new file mode 100644
index 0000000..34fa920
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/TableSchemaBuilderSuite.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TableSchemaBuilderSuite {
+
+ @Test(expected = NullPointerException.class)
+ public void testNullField() {
+ TableSchemaBuilder builder = TableSchema.builder();
+ builder.addColumn(null, true);
+ }
+
+ @Test
+ public void testBuilder() {
+ TableSchemaBuilder builder = TableSchema.builder();
+ builder.addColumn(new StructField("a", DataTypes.INT), true);
+ builder.addColumn(new StructField("b", DataTypes.DOUBLE), false);
+ TableSchema schema = builder.build();
+ Assert.assertEquals(2, schema.getListOfColumns().size());
+ List<ColumnSchema> columns = schema.getListOfColumns();
+ Assert.assertEquals("a", columns.get(0).getColumnName());
+ Assert.assertEquals("b", columns.get(1).getColumnName());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRepeatedColumn() {
+ TableSchemaBuilder builder = TableSchema.builder();
+ builder.addColumn(new StructField("a", DataTypes.INT), true);
+ builder.addColumn(new StructField("a", DataTypes.DOUBLE), false);
+ TableSchema schema = builder.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 49e4420..8d394db 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -217,6 +217,51 @@ object DataLoadingUtil {
/**
* build CarbonLoadModel for data loading
+ * @param table CarbonTable object containing all metadata information for the table
+ * like table name, table path, schema, etc
+ * @param options Load options from user input
+ * @return a new CarbonLoadModel instance
+ */
+ def buildCarbonLoadModelJava(
+ table: CarbonTable,
+ options: java.util.Map[String, String]
+ ): CarbonLoadModel = {
+ val carbonProperty: CarbonProperties = CarbonProperties.getInstance
+ val optionsFinal = getDataLoadingOptions(carbonProperty, options.asScala.toMap)
+ optionsFinal.put("sort_scope", "no_sort")
+ if (!options.containsKey("fileheader")) {
+ val csvHeader = table.getCreateOrderColumn(table.getTableName)
+ .asScala.map(_.getColName).mkString(",")
+ optionsFinal.put("fileheader", csvHeader)
+ }
+ val model = new CarbonLoadModel()
+ buildCarbonLoadModel(
+ table = table,
+ carbonProperty = carbonProperty,
+ options = options.asScala.toMap,
+ optionsFinal = optionsFinal,
+ carbonLoadModel = model,
+ hadoopConf = null) // we have provided 'fileheader', so it can be null
+
+ // set default values
+ model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
+ model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
+ model.setUseOnePass(options.asScala.getOrElse("onepass", "false").toBoolean)
+ model.setDictionaryServerHost(options.asScala.getOrElse("dicthost", null))
+ model.setDictionaryServerPort(options.asScala.getOrElse("dictport", "-1").toInt)
+ model
+ }
+
+ /**
+ * build CarbonLoadModel for data loading
+ * @param table CarbonTable object containing all metadata information for the table
+ * like table name, table path, schema, etc
+ * @param carbonProperty Carbon property instance
+ * @param options Load options from user input
+ * @param optionsFinal Load options that populated with default values for optional options
+ * @param carbonLoadModel The output load model
+ * @param hadoopConf hadoopConf is needed to read CSV header if there 'fileheader' is not set in
+ * user provided load options
*/
def buildCarbonLoadModel(
table: CarbonTable,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0b9917d..520d770 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,6 +99,7 @@
<module>core</module>
<module>processing</module>
<module>hadoop</module>
+ <module>store/sdk</module>
<module>integration/spark-common</module>
<module>integration/spark-common-test</module>
<module>assembly</module>
@@ -571,6 +572,12 @@
<id>include-all</id>
</profile>
<profile>
+ <id>store-sdk</id>
+ <modules>
+ <module>store/sdk</module>
+ </modules>
+ </profile>
+ <profile>
<id>sdvtest</id>
<modules>
<module>integration/spark-common-cluster-test</module>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
new file mode 100644
index 0000000..6663683
--- /dev/null
+++ b/store/sdk/pom.xml
@@ -0,0 +1,130 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>carbondata-store-sdk</artifactId>
+ <name>Apache CarbonData :: Store SDK</name>
+
+ <properties>
+ <dev.path>${basedir}/../../dev</dev.path>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.carbondata</groupId>
+ <artifactId>carbondata-spark-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <version>2.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/resources</directory>
+ </resource>
+ <resource>
+ <directory>.</directory>
+ <includes>
+ <include>CARBON_SPARK_INTERFACELogResource.properties</include>
+ </includes>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.scala-tools</groupId>
+ <artifactId>maven-scala-plugin</artifactId>
+ <version>2.15.2</version>
+ <executions>
+ <execution>
+ <id>compile</id>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <phase>compile</phase>
+ </execution>
+ <execution>
+ <id>testCompile</id>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <phase>test</phase>
+ </execution>
+ <execution>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.18</version>
+ <!-- Note config is repeated in scalatest config -->
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <argLine>-Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m</argLine>
+ <systemProperties>
+ <java.awt.headless>true</java.awt.headless>
+ </systemProperties>
+ <failIfNoTests>false</failIfNoTests>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>1.0</version>
+ <!-- Note config is repeated in surefire config -->
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <filereports>CarbonTestSuite.txt</filereports>
+ <argLine> ${argLine} -ea -Xmx3g -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
+ </argLine>
+ <stderr />
+ <environmentVariables>
+ </environmentVariables>
+ <systemProperties>
+ <java.awt.headless>true</java.awt.headless>
+ </systemProperties>
+ </configuration>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
new file mode 100644
index 0000000..daea733
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
+import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+/**
+ * Implementation to write rows in CSV format to carbondata file.
+ */
+@InterfaceAudience.Developer
+@InterfaceStability.Unstable
+class CSVCarbonWriter extends CarbonWriter {
+
+ private RecordWriter<NullWritable, StringArrayWritable> recordWriter;
+ private TaskAttemptContext context;
+ private StringArrayWritable writable;
+
+ CSVCarbonWriter(CarbonLoadModel loadModel) throws IOException {
+ Configuration hadoopConf = new Configuration();
+ CarbonTableOutputFormat.setLoadModel(hadoopConf, loadModel);
+ CarbonTableOutputFormat format = new CarbonTableOutputFormat();
+ JobID jobId = new JobID(UUID.randomUUID().toString(), 0);
+ Random random = new Random();
+ TaskID task = new TaskID(jobId, TaskType.MAP, random.nextInt());
+ TaskAttemptID attemptID = new TaskAttemptID(task, random.nextInt());
+ TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID);
+ this.recordWriter = format.getRecordWriter(context);
+ this.context = context;
+ this.writable = new StringArrayWritable();
+ }
+
+ /**
+ * Write single row data, input row is of type String[]
+ */
+ @Override
+ public void write(Object object) throws IOException {
+ writable.set((String[]) object);
+ try {
+ recordWriter.write(NullWritable.get(), writable);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * Flush and close the writer
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ recordWriter.close(context);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriter.java
new file mode 100644
index 0000000..e29aa18
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * Writer to write row data to carbondata file. Call {@link #builder()} to get
+ * a build to create instance of writer.
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public abstract class CarbonWriter {
+
+ /**
+ * Write an object to the file, the format of the object depends on the
+ * implementation
+ */
+ public abstract void write(Object object) throws IOException;
+
+ /**
+ * Flush and close the writer
+ */
+ public abstract void close() throws IOException;
+
+ /**
+ * Create a {@link CarbonWriterBuilder} to build a {@link CarbonWriter}
+ */
+ public static CarbonWriterBuilder builder() {
+ return new CarbonWriterBuilder();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
new file mode 100644
index 0000000..51ca09c
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.carbondata.common.Strings;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.converter.SchemaConverter;
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.metadata.schema.table.TableSchema;
+import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.ThriftWriter;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.spark.util.DataLoadingUtil;
+
+/**
+ * Biulder for {@link CarbonWriter}
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class CarbonWriterBuilder {
+ private Schema schema;
+ private String path;
+ private String[] sortColumns;
+ private boolean persistSchemaFile;
+
+ public CarbonWriterBuilder withSchema(Schema schema) {
+ Objects.requireNonNull(schema, "schema should not be null");
+ this.schema = schema;
+ return this;
+ }
+
+ public CarbonWriterBuilder outputPath(String path) {
+ Objects.requireNonNull(path, "path should not be null");
+ this.path = path;
+ return this;
+ }
+
+ public CarbonWriterBuilder sortBy(String[] sortColumns) {
+ this.sortColumns = sortColumns;
+ return this;
+ }
+
+ public CarbonWriterBuilder partitionBy(String[] partitionColumns) {
+ throw new UnsupportedOperationException();
+ }
+
+ public CarbonWriterBuilder persistSchemaFile(boolean persist) {
+ this.persistSchemaFile = persist;
+ return this;
+ }
+
+ public CarbonWriterBuilder withBlockSize(int blockSize) {
+ if (blockSize <= 0) {
+ throw new IllegalArgumentException("blockSize should be greater than zero");
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ public CarbonWriterBuilder withBlockletSize(int blockletSize) {
+ if (blockletSize <= 0) {
+ throw new IllegalArgumentException("blockletSize should be greater than zero");
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Build a {@link CSVCarbonWriter}, which accepts row in CSV format
+ */
+ public CarbonWriter buildWriterForCSVInput() throws IOException {
+ Objects.requireNonNull(schema, "schema should not be null");
+ Objects.requireNonNull(path, "path should not be null");
+
+ // build CarbonTable using schema
+ CarbonTable table = buildCarbonTable();
+ if (persistSchemaFile) {
+ // we are still using the traditional carbon table folder structure
+ persistSchemaFile(table, CarbonTablePath.getSchemaFilePath(path));
+ }
+
+ // build LoadModel
+ CarbonLoadModel loadModel = buildLoadModel(table);
+ return new CSVCarbonWriter(loadModel);
+ }
+
+ /**
+ * Build a {@link AvroCarbonWriter}, which accepts Avro object
+ * @return
+ * @throws IOException
+ */
+ public CarbonWriter buildWriterForAvroInput() throws IOException {
+ // TODO
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Build a {@link CarbonTable}
+ */
+ private CarbonTable buildCarbonTable() {
+ TableSchemaBuilder tableSchemaBuilder = TableSchema.builder();
+ List<String> sortColumnsList;
+ if (sortColumns != null) {
+ sortColumnsList = Arrays.asList(sortColumns);
+ } else {
+ sortColumnsList = new LinkedList<>();
+ }
+ for (Field field : schema.getFields()) {
+ tableSchemaBuilder.addColumn(
+ new StructField(field.getFieldName(), field.getDataType()),
+ sortColumnsList.contains(field.getFieldName()));
+ }
+ String tableName = "_tempTable";
+ String dbName = "_tempDB";
+ TableSchema schema = tableSchemaBuilder.build();
+ schema.setTableName(tableName);
+ CarbonTable table = CarbonTable.builder()
+ .tableName(schema.getTableName())
+ .databaseName(dbName)
+ .tablePath(path)
+ .tableSchema(schema)
+ .build();
+ return table;
+ }
+
+ /**
+ * Save the schema of the {@param table} to {@param persistFilePath}
+ * @param table table object containing schema
+ * @param persistFilePath absolute file path with file name
+ */
+ private void persistSchemaFile(CarbonTable table, String persistFilePath) throws IOException {
+ TableInfo tableInfo = table.getTableInfo();
+ String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(persistFilePath);
+ CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ org.apache.carbondata.format.TableInfo thriftTableInfo =
+ schemaConverter.fromWrapperToExternalTableInfo(
+ tableInfo,
+ tableInfo.getDatabaseName(),
+ tableInfo.getFactTable().getTableName());
+ org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry =
+ new org.apache.carbondata.format.SchemaEvolutionEntry(
+ tableInfo.getLastUpdatedTime());
+ thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history()
+ .add(schemaEvolutionEntry);
+ FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
+ if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
+ FileFactory.mkdirs(schemaMetadataPath, fileType);
+ }
+ ThriftWriter thriftWriter = new ThriftWriter(persistFilePath, false);
+ thriftWriter.open();
+ thriftWriter.write(thriftTableInfo);
+ thriftWriter.close();
+ }
+
+ /**
+ * Build a {@link CarbonLoadModel}
+ */
+ private CarbonLoadModel buildLoadModel(CarbonTable table) {
+ Map<String, String> options = new HashMap<>();
+ if (sortColumns != null) {
+ options.put("sort_columns", Strings.mkString(sortColumns, ","));
+ }
+ return DataLoadingUtil.buildCarbonLoadModelJava(table, options);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
new file mode 100644
index 0000000..6742fa7
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+/**
+ * A field represent one column
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class Field {
+
+ private String name;
+ private DataType type;
+
+ public Field(String name, String type) {
+ this.name = name;
+ if (type.equalsIgnoreCase("string")) {
+ this.type = DataTypes.STRING;
+ } else if (type.equalsIgnoreCase("date")) {
+ this.type = DataTypes.DATE;
+ } else if (type.equalsIgnoreCase("timestamp")) {
+ this.type = DataTypes.TIMESTAMP;
+ } else if (type.equalsIgnoreCase("boolean")) {
+ this.type = DataTypes.BOOLEAN;
+ } else if (type.equalsIgnoreCase("byte")) {
+ this.type = DataTypes.BYTE;
+ } else if (type.equalsIgnoreCase("short")) {
+ this.type = DataTypes.SHORT;
+ } else if (type.equalsIgnoreCase("int")) {
+ this.type = DataTypes.INT;
+ } else if (type.equalsIgnoreCase("long")) {
+ this.type = DataTypes.LONG;
+ } else if (type.equalsIgnoreCase("float")) {
+ this.type = DataTypes.FLOAT;
+ } else if (type.equalsIgnoreCase("double")) {
+ this.type = DataTypes.DOUBLE;
+ } else {
+ throw new IllegalArgumentException("unsupported data type: " + type);
+ }
+ }
+
+ public Field(String name, DataType type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getFieldName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
new file mode 100644
index 0000000..52a4611
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
+
+/**
+ * A schema used to write and read data files
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class Schema {
+
+ private Field[] fields;
+
+ public Schema(Field[] fields) {
+ this.fields = fields;
+ }
+
+ /**
+ * Create a Schema using JSON string, for example:
+ * [
+ * {"name":"string"},
+ * {"age":"int"}
+ * ]
+ */
+ public static Schema parseJson(String json) {
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(Field.class, new TypeAdapter<Field>() {
+ @Override
+ public void write(JsonWriter out, Field field) throws IOException {
+ // noop
+ }
+
+ @Override
+ public Field read(JsonReader in) throws IOException {
+ in.beginObject();
+ Field field = new Field(in.nextName(), in.nextString());
+ in.endObject();
+ return field;
+ }
+ });
+
+ Field[] fields = gsonBuilder.create().fromJson(json, Field[].class);
+ return new Schema(fields);
+ }
+
+ public Field[] getFields() {
+ return fields;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d827c7b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
new file mode 100644
index 0000000..531ec7c
--- /dev/null
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterSuite.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test suite for {@link CSVCarbonWriter}
+ */
+public class CSVCarbonWriterSuite {
+
+ @Test
+ public void testWriteFiles() throws IOException {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[2];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("age", DataTypes.INT);
+
+ writeFilesAndVerify(new Schema(fields), path);
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testWriteFilesJsonSchema() throws IOException {
+ String path = "./testWriteFilesJsonSchema";
+ FileUtils.deleteDirectory(new File(path));
+
+ String schema = new StringBuilder()
+ .append("[ \n")
+ .append(" {\"name\":\"string\"},\n")
+ .append(" {\"age\":\"int\"},\n")
+ .append(" {\"height\":\"double\"}\n")
+ .append("]")
+ .toString();
+
+ writeFilesAndVerify(Schema.parseJson(schema), path);
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ private void writeFilesAndVerify(Schema schema, String path) {
+ try {
+ CarbonWriter writer = CarbonWriter.builder()
+ .withSchema(schema)
+ .outputPath(path)
+ .buildWriterForCSVInput();
+
+ for (int i = 0; i < 100; i++) {
+ writer.write(new String[]{"robot" + i, String.valueOf(i), String.valueOf((double) i / 2)});
+ }
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+
+ File segmentFolder = new File(CarbonTablePath.getSegmentPath(path, "null"));
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertEquals(1, dataFiles.length);
+ }
+
+ @Test
+ public void testAllPrimitiveDataType() {
+ // TODO: write all data type and read by CarbonRecordReader to verify the content
+ }
+
+ @Test
+ public void test2Blocklet() {
+ // TODO: write data with more than one blocklet
+ }
+
+ @Test
+ public void test2Block() {
+ // TODO: write data with more than one block
+ }
+
+ @Test
+ public void testSortColumns() {
+ // TODO: test sort column
+ }
+
+ @Test
+ public void testPartitionOutput() {
+ // TODO: test write data with partition
+ }
+
+ @Test
+ public void testSchemaPersistence() {
+ // TODO: verify schema file is persisted in specified location
+ }
+
+}
[06/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 447ab46..547ecaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -35,12 +35,11 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -73,7 +72,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
ifDefaultValueMatchesFilter();
- if (isDimensionPresentInCurrentBlock[0] == true) {
+ if (isDimensionPresentInCurrentBlock[0]) {
isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
&& dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
}
@@ -120,11 +119,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
boolean isScanRequired = false;
if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
if (isMeasurePresentInCurrentBlock[0]) {
- minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ minValue = blockMinValue[measureChunkIndex[0] + lastDimensionColOrdinal];
isScanRequired =
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
- minValue = blockMinValue[dimensionBlocksIndex[0]];
+ minValue = blockMinValue[dimensionChunkIndex[0]];
isScanRequired = isScanRequired(minValue, filterRangeValues);
}
} else {
@@ -170,67 +169,69 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, true);
}
if (isDimensionPresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
- } else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ } else {
+ int chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
msrColEvalutorInfoList.get(0).getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(rawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
}
- return null;
}
@Override
@@ -280,7 +281,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
byte[] defaultValue = null;
if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -289,7 +290,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
dimColEvaluatorInfoList.get(0).getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
+ segmentProperties.getDimensions().get(dimensionChunkIndex[0]);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -300,14 +301,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
}
BitSet bitSet = null;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows,
+ if (dimensionColumnPage.isExplicitSorted()) {
+ bitSet = setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows,
defaultValue);
} else {
- bitSet = setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ bitSet = setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows, defaultValue);
}
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet,
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet,
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
}
return bitSet;
@@ -319,12 +320,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows,
+ DimensionColumnPage dimensionColumnPage, int numerOfRows,
byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
@@ -336,7 +337,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -354,7 +355,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
for (int i = 0; i < filterValues.length; i++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i], false);
if (start >= 0) {
// Logic will handle the case where the range filter member is not present in block
@@ -362,7 +363,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// set inorder to apply filters. this is Lesser than filter so the range will be taken
// from the prev element which is Lesser than filter member.
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[i]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[i]);
}
if (start < 0) {
start = -(start + 1);
@@ -373,14 +374,14 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0) {
start = start - 1;
}
}
last = start;
for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
last--;
}
startIndex = last;
@@ -397,11 +398,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
* be called if the column is sorted default so column index
* mapping will be present for accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows, byte[] defaultValue) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
@@ -414,7 +415,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
//find the number of default values to skip the null value in case of direct dictionary
if (null != defaultValue) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, defaultValue, true);
if (start < 0) {
skip = -(start + 1);
@@ -431,11 +432,11 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
for (int k = 0; k < filterValues.length; k++) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex,
numerOfRows - 1, filterValues[k], false);
if (start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[k]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[k]);
}
if (start < 0) {
start = -(start + 1);
@@ -446,7 +447,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
// When negative value of start is returned from getFirstIndexUsingBinarySearch the Start
// will be pointing to the next consecutive position. So compare it again and point to the
// previous value returned from getFirstIndexUsingBinarySearch.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
+ if (ByteUtil.compare(filterValues[k], dimensionColumnPage.getChunkData(start)) < 0) {
start = start - 1;
}
}
@@ -463,7 +464,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
} else {
for (int k = 0; k < filterValues.length; k++) {
for (int i = 0; i < numerOfRows; i++) {
- if (ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValues[k]) < 0) {
+ if (ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValues[k]) < 0) {
bitSet.set(i);
}
}
@@ -472,21 +473,24 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock[0]) {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
+ super.readColumnChunks(rawBlockletColumnChunks);
}
- int blockIndex = dimensionBlocksIndex[0];
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = dimensionChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock[0]) {
- int blockIndex = measureBlocksIndex[0];
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ int chunkIndex = measureChunkIndex[0];
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
index 4b3738a..2cbb234 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -22,7 +22,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class TrueFilterExecutor implements FilterExecuter {
@@ -33,19 +33,20 @@ public class TrueFilterExecutor implements FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
BitSetGroup group = new BitSetGroup(numberOfPages);
for (int i = 0; i < numberOfPages; i++) {
BitSet set = new BitSet();
- set.flip(0, blockChunkHolder.getDataBlock().getPageRowCount(i));
+ set.flip(0, rawBlockletColumnChunks.getDataBlock().getPageRowCount(i));
group.setBitSet(set, i);
}
return group;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
return true;
}
@@ -66,9 +67,9 @@ public class TrueFilterExecutor implements FilterExecuter {
/**
* It just reads necessary block for filter executor, it does not uncompress the data.
*
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
*/
- public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
// do nothing
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 516447f..bc3b39c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -21,27 +21,29 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class ValueBasedFilterExecuterImpl implements FilterExecuter {
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
-
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
return new BitSetGroup(0);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
throw new FilterUnsupportedException("Unsupported ValueBasedFilterExecuterImpl on row");
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return new BitSet(1);
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
deleted file mode 100644
index b3ed34e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.filter.intf;
-
-import org.apache.carbondata.core.scan.expression.Expression;
-
-public class FilterOptimizerBasic implements FilterOptimizer {
- @Override public Expression optimizeFilter() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
index e5cb0ad..bbd33ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
@@ -19,14 +19,13 @@ package org.apache.carbondata.core.scan.filter.optimizer;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl;
public class RangeFilterOptmizer extends FilterOptimizerImpl {
RangeExpressionEvaluator rangeExpEvaluator;
- public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) {
+ public RangeFilterOptmizer(Expression filterExpression) {
super(filterExpression);
this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index bb66d18..996965d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -46,17 +46,13 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
protected boolean isIncludeFilter;
private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
private MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo;
- private AbsoluteTableIdentifier tableIdentifier;
- private boolean isMeasure;
public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier, boolean isMeasure) {
+ boolean isIncludeFilter, boolean isMeasure) {
this.exp = exp;
this.isExpressionResolve = isExpressionResolve;
this.isIncludeFilter = isIncludeFilter;
- this.tableIdentifier = tableIdentifier;
- this.isMeasure = isMeasure;
- if (isMeasure == false) {
+ if (!isMeasure) {
this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
} else {
this.msrColResolvedFilterInfo = new MeasureColumnResolvedFilterInfo();
@@ -230,10 +226,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
return msrColResolvedFilterInfo;
}
- public AbsoluteTableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
/**
* method will calculates the start key based on the filter surrogates
*/
@@ -245,12 +237,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
setOfStartKeyByteArray);
}
-// else {
-// FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
-// segmentProperties, startKey, startKeyList);
-// FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
-// setOfStartKeyByteArray);
-// }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index 33bafb1..c846c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -40,7 +40,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 0e7666c..e85e9d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -61,7 +61,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
index f8586ed..4165424 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -77,10 +77,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
}
}
- public Map<CarbonMeasure, List<ColumnFilterInfo>> getMeasureResolvedFilterInstance() {
- return measureResolvedFilter;
- }
-
public ColumnFilterInfo getFilterValues() {
return resolvedFilterValueObj;
}
@@ -109,10 +105,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
this.type = dataType;
}
- public boolean isMeasureExistsInCurrentSlice() {
- return true;
- }
-
public CarbonColumn getCarbonColumn() {
return carbonColumn;
}
@@ -125,10 +117,6 @@ public class MeasureColumnResolvedFilterInfo extends ColumnResolvedFilterInfo
return carbonMeasure;
}
- public boolean isMeasureExistsInCurrentSilce() {
- return isMeasureExistsInCurrentSilce;
- }
-
public void setMeasureExistsInCurrentSilce(boolean measureExistsInCurrentSilce) {
isMeasureExistsInCurrentSilce = measureExistsInCurrentSilce;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
index d403ef4..104f69e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
@@ -28,9 +28,9 @@ import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolver
public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
public TrueConditionalResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+ boolean isIncludeFilter) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
+ super(exp, isExpressionResolve, isIncludeFilter, false);
}
@Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier,
@@ -47,16 +47,4 @@ public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
return FilterExecuterType.TRUE;
}
- /**
- * Method will the read filter expression corresponding to the resolver.
- * This method is required in row level executer inorder to evaluate the filter
- * expression against spark, as mentioned above row level is a special type
- * filter resolver.
- *
- * @return Expression
- */
- public Expression getFilterExpresion() {
- return exp;
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
index c90816a..b1e5a7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -19,7 +19,6 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
import java.io.IOException;
import java.util.Collections;
-import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
@@ -47,7 +46,6 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
if (visitableObj instanceof DimColumnResolvedFilterInfo) {
DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
ColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
resolvedFilterObject = FilterUtil
.getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
metadata.getColumnExpression(), metadata.isIncludeFilter(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
deleted file mode 100644
index 5a5b042..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.scan.expression.Expression;
-
-/**
- * This class contains all the logical information about the query like dimensions,measures,
- * sort order, topN etc..
- */
-public class CarbonQueryPlan implements Serializable {
- /**
- *
- */
- private static final long serialVersionUID = -9036044826928017164L;
-
- /**
- * Database name
- */
- private String databaseName;
-
- /**
- * Table name
- */
- private String tableName;
-
- /**
- * List of dimensions.
- * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
- * and department_name are dimensions
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryDimension> dimensions =
- new ArrayList<QueryDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * List of measures.
- * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
- * would be measure.
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryMeasure> measures =
- new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * expression
- */
- private Expression expression;
-
- /**
- * queryId
- */
- private String queryId;
-
- /**
- * If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data
- * with out decoding.
- */
- private boolean rawDetailQuery;
-
- /**
- * Constructor created with database name and table name.
- *
- * @param databaseName
- * @param tableName
- */
- public CarbonQueryPlan(String databaseName, String tableName) {
- this.tableName = tableName;
- this.databaseName = databaseName;
- }
-
- /**
- * @return the dimensions
- */
- public List<QueryDimension> getDimensions() {
- return dimensions;
- }
-
- public void addDimension(QueryDimension dimension) {
- this.dimensions.add(dimension);
- }
-
- /**
- * @return the measures
- */
- public List<QueryMeasure> getMeasures() {
- return measures;
- }
-
- public void addMeasure(QueryMeasure measure) {
- this.measures.add(measure);
- }
-
- public Expression getFilterExpression() {
- return expression;
- }
-
- public void setFilterExpression(Expression expression) {
- this.expression = expression;
- }
-
- /**
- * @return the databaseName
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * @return the tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- public String getQueryId() {
- return queryId;
- }
-
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- public boolean isRawDetailQuery() {
- return rawDetailQuery;
- }
-
- public void setRawDetailQuery(boolean rawDetailQuery) {
- this.rawDetailQuery = rawDetailQuery;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
new file mode 100644
index 0000000..0768221
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionColumn.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.model;
+
+/**
+ * Contains information for a column for projection
+ */
+public class ProjectionColumn {
+
+ /**
+ * name of the column
+ */
+ protected String columnName;
+
+ /**
+ * query order in which result of the query will be send
+ */
+ private int projectionOrdinal;
+
+ ProjectionColumn(String columnName) {
+ this.columnName = columnName;
+ }
+
+ /**
+ * @return the columnName
+ */
+ public String getColumnName() {
+ return columnName;
+ }
+
+ /**
+ * @return the projectionOrdinal
+ */
+ public int getOrdinal() {
+ return projectionOrdinal;
+ }
+
+ /**
+ * @param projectionOrdinal the projectionOrdinal to set
+ */
+ public void setOrdinal(int projectionOrdinal) {
+ this.projectionOrdinal = projectionOrdinal;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
new file mode 100644
index 0000000..806de85
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionDimension.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+
+/**
+ * dimension column which holds the information about the column
+ * this is done to avoid heavy object serialization
+ */
+public class ProjectionDimension extends ProjectionColumn {
+
+ /**
+ * actual dimension column
+ */
+ private CarbonDimension dimension;
+
+ public ProjectionDimension(CarbonDimension dimension) {
+ super(dimension.getColName());
+ this.dimension = dimension;
+ }
+
+ /**
+ * @return the dimension
+ */
+ public CarbonDimension getDimension() {
+ return dimension;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
new file mode 100644
index 0000000..93d9c82
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/ProjectionMeasure.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+
+/**
+ * query plan measure, this class will holds the information
+ * about measure present in the query, this is done to avoid the serialization
+ * of the heavy object
+ */
+public class ProjectionMeasure extends ProjectionColumn {
+
+ /**
+ * actual carbon measure object
+ */
+ private CarbonMeasure measure;
+
+ public ProjectionMeasure(CarbonMeasure measure) {
+ super(measure.getColName());
+ this.measure = measure;
+ }
+
+ /**
+ * @return the measure
+ */
+ public CarbonMeasure getMeasure() {
+ return measure;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
deleted file mode 100644
index 1336b87..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-/**
- * query column which will have information about column
- */
-public class QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -4222306600480181084L;
-
- /**
- * name of the column
- */
- protected String columnName;
-
- /**
- * query order in which result of the query will be send
- */
- private int queryOrder;
-
- public QueryColumn(String columnName) {
- this.columnName = columnName;
- }
-
- /**
- * @return the columnName
- */
- public String getColumnName() {
- return columnName;
- }
-
- /**
- * @return the queryOrder
- */
- public int getQueryOrder() {
- return queryOrder;
- }
-
- /**
- * @param queryOrder the queryOrder to set
- */
- public void setQueryOrder(int queryOrder) {
- this.queryOrder = queryOrder;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
deleted file mode 100644
index e8f07a7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-
-/**
- * query plan dimension which will holds the information about the query plan dimension
- * this is done to avoid heavy object serialization
- */
-public class QueryDimension extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -8492704093776645651L;
- /**
- * actual dimension column
- */
- private transient CarbonDimension dimension;
-
- public QueryDimension(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the dimension
- */
- public CarbonDimension getDimension() {
- return dimension;
- }
-
- /**
- * @param dimension the dimension to set
- */
- public void setDimension(CarbonDimension dimension) {
- this.dimension = dimension;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
deleted file mode 100644
index 73a8ad5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-
-/**
- * query plan measure, this class will holds the information
- * about measure present in the query, this is done to avoid the serialization
- * of the heavy object
- */
-public class QueryMeasure extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = 1035512411375495414L;
-
- /**
- * actual carbon measure object
- */
- private transient CarbonMeasure measure;
-
- public QueryMeasure(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the measure
- */
- public CarbonMeasure getMeasure() {
- return measure;
- }
-
- /**
- * @param measure the measure to set
- */
- public void setMeasure(CarbonMeasure measure) {
- this.measure = measure;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 655ed5e..4e3665f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.scan.model;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -46,25 +45,17 @@ import org.apache.carbondata.core.util.DataTypeConverter;
* about the query, This will be sent from driver to executor '
* This will be refereed to executing the query.
*/
-public class QueryModel implements Serializable {
+public class QueryModel {
/**
- * serialization version
- */
- private static final long serialVersionUID = -4674677234007089052L;
- /**
* this will hold the information about the dictionary dimension
* which to
*/
- public transient Map<String, Dictionary> columnToDictionaryMapping;
- /**
- * list of dimension selected for in query
- */
- private List<QueryDimension> queryDimension;
+ private transient Map<String, Dictionary> columnToDictionaryMapping;
/**
- * list of measure selected in query
+ * list of projection columns in query
*/
- private List<QueryMeasure> queryMeasures;
+ private QueryProjection projection;
/**
* query id
*/
@@ -79,10 +70,6 @@ public class QueryModel implements Serializable {
*/
private List<TableBlockInfo> tableBlockInfos;
/**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
- /**
* To handle most of the computation in query engines like spark and hive, carbon should give
* raw detailed records to it.
*/
@@ -117,43 +104,15 @@ public class QueryModel implements Serializable {
*/
private boolean readPageByPage;
- public QueryModel() {
+ private QueryModel(CarbonTable carbonTable) {
tableBlockInfos = new ArrayList<TableBlockInfo>();
- queryDimension = new ArrayList<QueryDimension>();
- queryMeasures = new ArrayList<QueryMeasure>();
invalidSegmentIds = new ArrayList<>();
+ this.table = carbonTable;
+ this.queryId = String.valueOf(System.nanoTime());
}
- public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier,
- CarbonQueryPlan queryPlan, CarbonTable carbonTable, DataTypeConverter converter) {
- QueryModel queryModel = new QueryModel();
- queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier);
-
- fillQueryModel(queryPlan, carbonTable, queryModel);
-
- queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery());
- queryModel.setQueryId(queryPlan.getQueryId());
- queryModel.setConverter(converter);
- return queryModel;
- }
-
- private static void fillQueryModel(CarbonQueryPlan queryPlan, CarbonTable carbonTable,
- QueryModel queryModel) {
- queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- queryModel.setQueryDimension(queryPlan.getDimensions());
- queryModel.setQueryMeasures(queryPlan.getMeasures());
- if (null != queryPlan.getFilterExpression()) {
- boolean[] isFilterDimensions = new boolean[carbonTable.getDimensionOrdinalMax()];
- boolean[] isFilterMeasures =
- new boolean[carbonTable.getNumberOfMeasures(carbonTable.getTableName())];
- processFilterExpression(carbonTable, queryPlan.getFilterExpression(), isFilterDimensions,
- isFilterMeasures);
- queryModel.setIsFilterDimensions(isFilterDimensions);
- queryModel.setIsFilterMeasures(isFilterMeasures);
- }
- //TODO need to remove this code, and executor will load the table
- // from file metadata
- queryModel.setTable(carbonTable);
+ public static QueryModel newInstance(CarbonTable carbonTable) {
+ return new QueryModel(carbonTable);
}
public static void processFilterExpression(CarbonTable carbonTable, Expression filterExpression,
@@ -239,42 +198,26 @@ public class QueryModel implements Serializable {
*/
public CarbonColumn[] getProjectionColumns() {
CarbonColumn[] carbonColumns =
- new CarbonColumn[getQueryDimension().size() + getQueryMeasures().size()];
- for (QueryDimension dimension : getQueryDimension()) {
- carbonColumns[dimension.getQueryOrder()] = dimension.getDimension();
+ new CarbonColumn[getProjectionDimensions().size() + getProjectionMeasures().size()];
+ for (ProjectionDimension dimension : getProjectionDimensions()) {
+ carbonColumns[dimension.getOrdinal()] = dimension.getDimension();
}
- for (QueryMeasure msr : getQueryMeasures()) {
- carbonColumns[msr.getQueryOrder()] = msr.getMeasure();
+ for (ProjectionMeasure msr : getProjectionMeasures()) {
+ carbonColumns[msr.getOrdinal()] = msr.getMeasure();
}
return carbonColumns;
}
- /**
- * @return the queryDimension
- */
- public List<QueryDimension> getQueryDimension() {
- return queryDimension;
+ public void setProjection(QueryProjection projection) {
+ this.projection = projection;
}
- /**
- * @param queryDimension the queryDimension to set
- */
- public void setQueryDimension(List<QueryDimension> queryDimension) {
- this.queryDimension = queryDimension;
+ public List<ProjectionDimension> getProjectionDimensions() {
+ return projection.getDimensions();
}
- /**
- * @return the queryMeasures
- */
- public List<QueryMeasure> getQueryMeasures() {
- return queryMeasures;
- }
-
- /**
- * @param queryMeasures the queryMeasures to set
- */
- public void setQueryMeasures(List<QueryMeasure> queryMeasures) {
- this.queryMeasures = queryMeasures;
+ public List<ProjectionMeasure> getProjectionMeasures() {
+ return projection.getMeasures();
}
/**
@@ -285,13 +228,6 @@ public class QueryModel implements Serializable {
}
/**
- * @param queryId the queryId to set
- */
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- /**
* @return the tableBlockInfos
*/
public List<TableBlockInfo> getTableBlockInfos() {
@@ -320,14 +256,7 @@ public class QueryModel implements Serializable {
* @return the absoluteTableIdentifier
*/
public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- /**
- * @param absoluteTableIdentifier the absoluteTableIdentifier to set
- */
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
+ return table.getAbsoluteTableIdentifier();
}
/**
@@ -337,13 +266,6 @@ public class QueryModel implements Serializable {
return table;
}
- /**
- * @param table the table to set
- */
- public void setTable(CarbonTable table) {
- this.table = table;
- }
-
public boolean isForcedDetailRawQuery() {
return forcedDetailRawQuery;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
new file mode 100644
index 0000000..d4496e0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryProjection.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+
+/**
+ * Contains projection columns in the query
+ */
+public class QueryProjection {
+
+ /**
+ * List of dimensions.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
+ * and department_name are dimensions
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<ProjectionDimension> dimensions =
+ new ArrayList<ProjectionDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * List of measures.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
+ * would be measure.
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<ProjectionMeasure> measures =
+ new ArrayList<ProjectionMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * Constructor created with database name and table name.
+ *
+ */
+ public QueryProjection() {
+ }
+
+ /**
+ * @return the dimensions
+ */
+ public List<ProjectionDimension> getDimensions() {
+ return dimensions;
+ }
+
+ public void addDimension(CarbonDimension dimension, int queryOrdinal) {
+ ProjectionDimension queryDimension = new ProjectionDimension(dimension);
+ queryDimension.setOrdinal(queryOrdinal);
+ this.dimensions.add(queryDimension);
+ }
+
+ /**
+ * @return the measures
+ */
+ public List<ProjectionMeasure> getMeasures() {
+ return measures;
+ }
+
+ public void addMeasure(CarbonMeasure measure, int queryOrdinal) {
+ ProjectionMeasure queryMeasure = new ProjectionMeasure(measure);
+ queryMeasure.setOrdinal(queryOrdinal);
+ this.measures.add(queryMeasure);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
deleted file mode 100644
index eb5e3f9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.scan.collector.ResultCollectorFactory;
-import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
-import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.scan.scanner.BlockletScanner;
-import org.apache.carbondata.core.scan.scanner.impl.FilterScanner;
-import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.util.TaskMetricsMap;
-
-/**
- * This abstract class provides a skeletal implementation of the
- * Block iterator.
- */
-public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
-
- /**
- * iterator which will be used to iterate over data blocks
- */
- protected CarbonIterator<DataRefNode> dataBlockIterator;
-
- /**
- * result collector which will be used to aggregate the scanned result
- */
- protected ScannedResultCollector scannerResultAggregator;
-
- /**
- * processor which will be used to process the block processing can be
- * filter processing or non filter processing
- */
- protected BlockletScanner blockletScanner;
-
- /**
- * batch size of result
- */
- protected int batchSize;
-
- protected ExecutorService executorService;
-
- private Future<AbstractScannedResult> future;
-
- private Future<BlocksChunkHolder> futureIo;
-
- protected AbstractScannedResult scannedResult;
-
- private BlockExecutionInfo blockExecutionInfo;
-
- private FileHolder fileReader;
-
- private AtomicBoolean nextBlock;
-
- private AtomicBoolean nextRead;
-
- public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel, ExecutorService executorService) {
- this.blockExecutionInfo = blockExecutionInfo;
- this.fileReader = fileReader;
- dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
- blockExecutionInfo.getNumberOfBlockToScan());
- if (blockExecutionInfo.getFilterExecuterTree() != null) {
- blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
- } else {
- blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
- }
- this.scannerResultAggregator =
- ResultCollectorFactory.getScannedResultCollector(blockExecutionInfo);
- this.batchSize = batchSize;
- this.executorService = executorService;
- this.nextBlock = new AtomicBoolean(false);
- this.nextRead = new AtomicBoolean(false);
- }
-
- public boolean hasNext() {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- if (null != scannedResult) {
- scannedResult.freeMemory();
- }
- return dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get();
- }
- }
-
- protected boolean updateScanner() {
- try {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- scannedResult = getNextScannedResult();
- while (scannedResult != null) {
- if (scannedResult.hasNext()) {
- return true;
- }
- scannedResult = getNextScannedResult();
- }
- nextBlock.set(false);
- nextRead.set(false);
- return false;
- }
- } catch (Exception ex) {
- throw new RuntimeException(ex);
- }
- }
-
- private AbstractScannedResult getNextScannedResult() throws Exception {
- AbstractScannedResult result = null;
- if (blockExecutionInfo.isPrefetchBlocklet()) {
- if (dataBlockIterator.hasNext() || nextBlock.get() || nextRead.get()) {
- if (future == null) {
- future = execute();
- }
- result = future.get();
- nextBlock.set(false);
- if (dataBlockIterator.hasNext() || nextRead.get()) {
- nextBlock.set(true);
- future = execute();
- }
- }
- } else {
- if (dataBlockIterator.hasNext()) {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
- if (blocksChunkHolder != null) {
- result = blockletScanner.scanBlocklet(blocksChunkHolder);
- }
- }
- }
- return result;
- }
-
- private BlocksChunkHolder getBlocksChunkHolder() throws IOException {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolderInternal();
- while (blocksChunkHolder == null && dataBlockIterator.hasNext()) {
- blocksChunkHolder = getBlocksChunkHolderInternal();
- }
- return blocksChunkHolder;
- }
-
- private BlocksChunkHolder getBlocksChunkHolderInternal() throws IOException {
- BlocksChunkHolder blocksChunkHolder =
- new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
- blockExecutionInfo.getTotalNumberOfMeasureBlock(), fileReader);
- blocksChunkHolder.setDataBlock(dataBlockIterator.next());
- if (blocksChunkHolder.getDataBlock().getColumnsMaxValue() == null) {
- return blocksChunkHolder;
- }
- if (blockletScanner.isScanRequired(blocksChunkHolder)) {
- return blocksChunkHolder;
- }
- return null;
- }
-
- private Future<AbstractScannedResult> execute() {
- return executorService.submit(new Callable<AbstractScannedResult>() {
- @Override public AbstractScannedResult call() throws Exception {
- if (futureIo == null) {
- futureIo = executeRead();
- }
- BlocksChunkHolder blocksChunkHolder = futureIo.get();
- futureIo = null;
- nextRead.set(false);
- if (blocksChunkHolder != null) {
- if (dataBlockIterator.hasNext()) {
- nextRead.set(true);
- futureIo = executeRead();
- }
- return blockletScanner.scanBlocklet(blocksChunkHolder);
- }
- return null;
- }
- });
- }
-
- private Future<BlocksChunkHolder> executeRead() {
- return executorService.submit(new Callable<BlocksChunkHolder>() {
- @Override public BlocksChunkHolder call() throws Exception {
- try {
- TaskMetricsMap.getInstance().registerThreadCallback();
- if (dataBlockIterator.hasNext()) {
- BlocksChunkHolder blocksChunkHolder = getBlocksChunkHolder();
- if (blocksChunkHolder != null) {
- blockletScanner.readBlocklet(blocksChunkHolder);
- return blocksChunkHolder;
- }
- }
- return null;
- } finally {
- // update read bytes metrics for this thread
- TaskMetricsMap.getInstance().updateReadBytes(Thread.currentThread().getId());
- }
- }
- });
- }
-
- public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
-
- /**
- * Close the resources
- */
- public void close() {
- // free the current scanned result
- if (null != scannedResult && !scannedResult.hasNext()) {
- scannedResult.freeMemory();
- }
- // free any pre-fetched memory if present
- if (null != future) {
- try {
- AbstractScannedResult abstractScannedResult = future.get();
- if (abstractScannedResult != null) {
- abstractScannedResult.freeMemory();
- }
- } catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
index 3ea4e1d..f0d81ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
@@ -23,7 +23,7 @@ import org.apache.carbondata.core.datastore.DataRefNode;
/**
* Below class will be used to iterate over data block
*/
-public class BlockletIterator extends CarbonIterator<DataRefNode> {
+class BlockletIterator extends CarbonIterator<DataRefNode> {
/**
* data store block
*/
@@ -47,17 +47,18 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> {
* Constructor
*
* @param datablock first data block
- * @param totalNumberOfBlocksToScan total number of blocks to be scanned
+ * @param totalNumberOfBlockletToScan total number of blocklets to be scanned
*/
- public BlockletIterator(DataRefNode datablock, long totalNumberOfBlocksToScan) {
+ BlockletIterator(DataRefNode datablock, long totalNumberOfBlockletToScan) {
this.datablock = datablock;
- this.totalNumberOfBlocksToScan = totalNumberOfBlocksToScan;
+ this.totalNumberOfBlocksToScan = totalNumberOfBlockletToScan;
}
/**
* is all the blocks assigned to this iterator has been processed
*/
- @Override public boolean hasNext() {
+ @Override
+ public boolean hasNext() {
return hasNext;
}
@@ -67,7 +68,8 @@ public class BlockletIterator extends CarbonIterator<DataRefNode> {
* @return next data block
*
*/
- @Override public DataRefNode next() {
+ @Override
+ public DataRefNode next() {
// get the current blocks
DataRefNode datablockTemp = datablock;
// store the next data block
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
deleted file mode 100644
index 60090d0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.processor;
-
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
-import org.apache.carbondata.core.util.BitSetGroup;
-
-/**
- * Block chunk holder which will hold the dimension and
- * measure chunk
- */
-public class BlocksChunkHolder {
-
- /**
- * dimension column data chunk
- */
- private DimensionRawColumnChunk[] dimensionRawDataChunk;
-
- /**
- * measure column data chunk
- */
- private MeasureRawColumnChunk[] measureRawDataChunk;
-
- /**
- * file reader which will use to read the block from file
- */
- private FileHolder fileReader;
-
- /**
- * data block
- */
- private DataRefNode dataBlock;
-
- private BitSetGroup bitSetGroup;
-
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- }
-
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock,
- FileHolder fileReader) {
- dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
- measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
- this.fileReader = fileReader;
- }
-
- /**
- * @return the dimensionRawDataChunk
- */
- public DimensionRawColumnChunk[] getDimensionRawDataChunk() {
- return dimensionRawDataChunk;
- }
-
- /**
- * @param dimensionRawDataChunk the dimensionRawDataChunk to set
- */
- public void setDimensionRawDataChunk(DimensionRawColumnChunk[] dimensionRawDataChunk) {
- this.dimensionRawDataChunk = dimensionRawDataChunk;
- }
-
- /**
- * @return the measureRawDataChunk
- */
- public MeasureRawColumnChunk[] getMeasureRawDataChunk() {
- return measureRawDataChunk;
- }
-
- /**
- * @param measureRawDataChunk the measureRawDataChunk to set
- */
- public void setMeasureRawDataChunk(MeasureRawColumnChunk[] measureRawDataChunk) {
- this.measureRawDataChunk = measureRawDataChunk;
- }
-
- /**
- * @return the fileReader
- */
- public FileHolder getFileReader() {
- return fileReader;
- }
-
- /**
- * @param fileReader the fileReader to set
- */
- public void setFileReader(FileHolder fileReader) {
- this.fileReader = fileReader;
- }
-
- /**
- * @return the dataBlock
- */
- public DataRefNode getDataBlock() {
- return dataBlock;
- }
-
- /**
- * @param dataBlock the dataBlock to set
- */
- public void setDataBlock(DataRefNode dataBlock) {
- this.dataBlock = dataBlock;
- }
-
- /***
- * To reset the measure chunk and dimension chunk
- * array
- */
- public void reset() {
- for (int i = 0; i < measureRawDataChunk.length; i++) {
- this.measureRawDataChunk[i] = null;
- }
- for (int i = 0; i < dimensionRawDataChunk.length; i++) {
- this.dimensionRawDataChunk[i] = null;
- }
- }
-
- public BitSetGroup getBitSetGroup() {
- return bitSetGroup;
- }
-
- public void setBitSetGroup(BitSetGroup bitSetGroup) {
- this.bitSetGroup = bitSetGroup;
- }
-}
[32/54] [abbrv] carbondata git commit: [CARBONDATA-1114][Tests] Fix
bugs in tests in windows env
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
index 812db5c..8d5f3d4 100644
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
@@ -65,7 +65,7 @@ public class LocalFileLockTest {
Assert.assertTrue(localLock1.unlock());
Assert.assertTrue(localLock2.lock());
-
+ Assert.assertTrue(localLock2.unlock());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
index d89f10d..30d9da4 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormatTest.java
@@ -176,6 +176,7 @@ public class CSVInputFormatTest extends TestCase {
FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
Assert.assertTrue(job.waitForCompletion(true));
+ deleteOutput(output);
}
private void prepareConf(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index 54fba55..b3dd464 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.carbondata</groupId>
<artifactId>carbondata-parent</artifactId>
- <version>1.3.0-SNAPSHOT</version>
+ <version>1.4.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/859d71c1/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
index dc5696a..df6afc6 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
@@ -23,7 +23,7 @@ import java.util.UUID;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
-import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
+import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.hadoop.conf.Configuration;
@@ -42,9 +42,9 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
@InterfaceAudience.Internal
class CSVCarbonWriter extends CarbonWriter {
- private RecordWriter<NullWritable, StringArrayWritable> recordWriter;
+ private RecordWriter<NullWritable, ObjectArrayWritable> recordWriter;
private TaskAttemptContext context;
- private StringArrayWritable writable;
+ private ObjectArrayWritable writable;
CSVCarbonWriter(CarbonLoadModel loadModel) throws IOException {
Configuration hadoopConf = new Configuration();
@@ -57,7 +57,7 @@ class CSVCarbonWriter extends CarbonWriter {
TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID);
this.recordWriter = format.getRecordWriter(context);
this.context = context;
- this.writable = new StringArrayWritable();
+ this.writable = new ObjectArrayWritable();
}
/**
[10/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
index 0cb2918..099fffd 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/SegmentInfo.java
@@ -29,31 +29,12 @@ public class SegmentInfo implements Serializable {
private static final long serialVersionUID = -1749874611112709431L;
/**
- * number of column in the segment
- */
- private int numberOfColumns;
-
- /**
* cardinality of each columns
* column which is not participating in the multidimensional key cardinality will be -1;
*/
private int[] columnCardinality;
/**
- * @return the numberOfColumns
- */
- public int getNumberOfColumns() {
- return numberOfColumns;
- }
-
- /**
- * @param numberOfColumns the numberOfColumns to set
- */
- public void setNumberOfColumns(int numberOfColumns) {
- this.numberOfColumns = numberOfColumns;
- }
-
- /**
* @return the columnCardinality
*/
public int[] getColumnCardinality() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6036569..d17d865 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -19,7 +19,13 @@ package org.apache.carbondata.core.metadata.schema.table;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -33,7 +39,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryProjection;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeConverter;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -136,10 +145,7 @@ public class CarbonTable implements Serializable {
/**
* During creation of TableInfo from hivemetastore the DataMapSchemas and the columns
* DataTypes are not converted to the appropriate child classes.
- *
* This method will cast the same to the appropriate classes
- *
- * @param tableInfo
*/
public static void updateTableInfo(TableInfo tableInfo) {
List<DataMapSchema> dataMapSchemas = new ArrayList<>();
@@ -153,8 +159,9 @@ public class CarbonTable implements Serializable {
}
tableInfo.setDataMapSchemaList(dataMapSchemas);
for (ColumnSchema columnSchema : tableInfo.getFactTable().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale()));
}
List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList();
for (DataMapSchema dataMapSchema : childSchema) {
@@ -168,10 +175,11 @@ public class CarbonTable implements Serializable {
}
}
if (tableInfo.getFactTable().getBucketingInfo() != null) {
- for (ColumnSchema columnSchema : tableInfo.getFactTable()
- .getBucketingInfo().getListOfColumns()) {
- columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
- columnSchema.getPrecision(), columnSchema.getScale()));
+ for (ColumnSchema columnSchema :
+ tableInfo.getFactTable().getBucketingInfo().getListOfColumns()) {
+ columnSchema.setDataType(
+ DataTypeUtil.valueOf(
+ columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale()));
}
}
if (tableInfo.getFactTable().getPartitionInfo() != null) {
@@ -211,6 +219,7 @@ public class CarbonTable implements Serializable {
/**
* fill columns as per user provided order
+ *
* @param tableName
*/
private void fillCreateOrderColumn(String tableName) {
@@ -229,7 +238,6 @@ public class CarbonTable implements Serializable {
this.createOrderColumn.put(tableName, columns);
}
-
/**
* Fill allDimensions and allMeasures for carbon table
*
@@ -254,8 +262,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, ++complexTypeOrdinal);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
allDimensions.add(complexDimension);
dimensionOrdinal =
@@ -268,9 +276,8 @@ public class CarbonTable implements Serializable {
this.numberOfSortColumns++;
}
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
- CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), -1, -1, -1);
if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
this.numberOfNoDictSortColumns++;
}
@@ -278,25 +285,25 @@ public class CarbonTable implements Serializable {
primitiveDimensions.add(dimension);
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
&& columnSchema.getColumnGroupId() == -1) {
- CarbonDimension dimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1);
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
+ -1, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else {
columnGroupOrdinal =
previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0;
previousColumnGroupId = columnSchema.getColumnGroupId();
- CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), keyOrdinal++,
+ CarbonDimension dimension = new CarbonDimension(
+ columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(), keyOrdinal++,
columnGroupOrdinal, -1);
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
}
}
} else {
- allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++,
- columnSchema.getSchemaOrdinal()));
+ allMeasures.add(
+ new CarbonMeasure(columnSchema, measureOrdinal++, columnSchema.getSchemaOrdinal()));
}
}
fillVisibleDimensions(tableSchema.getTableName());
@@ -347,8 +354,8 @@ public class CarbonTable implements Serializable {
if (columnSchema.isDimensionColumn()) {
if (columnSchema.getNumberOfChild() > 0) {
CarbonDimension complexDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, -1);
complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
parentDimension.getListOfChildDimensions().add(complexDimension);
dimensionOrdinal =
@@ -356,8 +363,8 @@ public class CarbonTable implements Serializable {
listOfColumns, complexDimension, primitiveDimensions);
} else {
CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, dimensionOrdinal++,
- columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ new CarbonDimension(columnSchema, dimensionOrdinal++, columnSchema.getSchemaOrdinal(),
+ -1, -1, -1);
parentDimension.getListOfChildDimensions().add(carbonDimension);
primitiveDimensions.add(carbonDimension);
}
@@ -569,6 +576,7 @@ public class CarbonTable implements Serializable {
}
return null;
}
+
/**
* gets all children dimension for complex type
*
@@ -771,18 +779,13 @@ public class CarbonTable implements Serializable {
return dimensionOrdinalMax;
}
- public void setDimensionOrdinalMax(int dimensionOrdinalMax) {
- this.dimensionOrdinalMax = dimensionOrdinalMax;
- }
-
-
public boolean hasDataMapSchema() {
return hasDataMapSchema;
}
public boolean isChildDataMap() {
- return null != tableInfo.getParentRelationIdentifiers()
- && !tableInfo.getParentRelationIdentifiers().isEmpty();
+ return null != tableInfo.getParentRelationIdentifiers() &&
+ !tableInfo.getParentRelationIdentifiers().isEmpty();
}
/**
@@ -806,4 +809,63 @@ public class CarbonTable implements Serializable {
}
return dataSize + indexSize;
}
+
+ /**
+ * Create a new QueryModel with projection all columns in the table.
+ */
+ public QueryModel createQueryModelWithProjectAllColumns(DataTypeConverter converter) {
+ QueryProjection projection = new QueryProjection();
+
+ List<CarbonDimension> dimensions = getDimensionByTableName(getTableName());
+ for (int i = 0; i < dimensions.size(); i++) {
+ projection.addDimension(dimensions.get(i), i);
+ }
+ List<CarbonMeasure> measures = getMeasureByTableName(getTableName());
+ for (int i = 0; i < measures.size(); i++) {
+ projection.addMeasure(measures.get(i), i);
+ }
+ QueryModel model = QueryModel.newInstance(this);
+ model.setProjection(projection);
+ model.setConverter(converter);
+ return model;
+ }
+
+ /**
+ * Create a new QueryModel with specified projection
+ */
+ public QueryModel createQueryWithProjection(String[] projectionColumnNames,
+ DataTypeConverter converter) {
+ QueryProjection projection = createProjection(projectionColumnNames);
+ QueryModel queryModel = QueryModel.newInstance(this);
+ queryModel.setProjection(projection);
+ queryModel.setConverter(converter);
+ return queryModel;
+ }
+
+ private QueryProjection createProjection(String[] projectionColumnNames) {
+ String factTableName = getTableName();
+ QueryProjection projection = new QueryProjection();
+ // fill dimensions
+ // If columns are null, set all dimensions and measures
+ int i = 0;
+ if (projectionColumnNames != null) {
+ for (String projectionColumnName : projectionColumnNames) {
+ CarbonDimension dimension = getDimensionByName(factTableName, projectionColumnName);
+ if (dimension != null) {
+ projection.addDimension(dimension, i);
+ i++;
+ } else {
+ CarbonMeasure measure = getMeasureByName(factTableName, projectionColumnName);
+ if (measure == null) {
+ throw new RuntimeException(projectionColumnName +
+ " column not found in the table " + factTableName);
+ }
+ projection.addMeasure(measure, i);
+ i++;
+ }
+ }
+ }
+
+ return projection;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
index 20e539b..9a1dad1 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
@@ -26,18 +26,6 @@ import java.io.Serializable;
*/
public class RelationIdentifier implements Serializable, Writable {
- public void setDatabaseName(String databaseName) {
- this.databaseName = databaseName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public void setTableId(String tableId) {
- this.tableId = tableId;
- }
-
private String databaseName;
private String tableName;
@@ -50,10 +38,6 @@ public class RelationIdentifier implements Serializable, Writable {
this.tableId = tableId;
}
- public RelationIdentifier() {
- this(null, null, null);
- }
-
public String getDatabaseName() {
return databaseName;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 4deafd4..0d796c7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -24,7 +24,11 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
index 13e7d49..c888418 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
@@ -45,7 +45,7 @@ public class CarbonColumn implements Serializable {
/**
* order in which user has created table
*/
- protected int schemaOrdinal;
+ private int schemaOrdinal;
/**
* Column identifier
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
index b4c052f..d15cde7 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonDimension.java
@@ -122,18 +122,6 @@ public class CarbonDimension extends CarbonColumn {
return getEncoder().contains(Encoding.DICTIONARY);
}
- public int getNumDimensionsExpanded() {
- if (listOfChildDimensions == null) {
- // there is no child, return 1 column
- return 1;
- }
- int columnCount = 1;
- for (CarbonDimension dimension: listOfChildDimensions) {
- columnCount += dimension.getNumDimensionsExpanded();
- }
- return columnCount;
- }
-
/**
* @return is column participated in sorting or not
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 18eae11..7289c98 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -128,8 +128,6 @@ public class CarbonUpdateUtil {
lockStatus = updateLock.lockWithRetries();
if (lockStatus) {
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
// read the existing file if present and update the same.
SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager
.getUpdateStatusDetails();
@@ -389,16 +387,6 @@ public class CarbonUpdateUtil {
.substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
}
- /**
- * returns segment id from segment name
- *
- * @param segmentName
- * @return
- */
- public static String getSegmentId(String segmentName) {
- return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
- }
-
public static long getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
String segmentDirPath = tablePath.getCarbonDataDirectoryPath(segmentId);
@@ -523,7 +511,7 @@ public class CarbonUpdateUtil {
// aborted scenario.
invalidDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
+ .getDeleteDeltaInvalidFilesList(block, false,
allSegmentFiles, isAbortedFile);
for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
boolean doForceDelete = true;
@@ -533,7 +521,7 @@ public class CarbonUpdateUtil {
// case 1
if (CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
completeListOfDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, true,
+ .getDeleteDeltaInvalidFilesList(block, true,
allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : completeListOfDeleteDeltaFiles) {
@@ -541,7 +529,7 @@ public class CarbonUpdateUtil {
}
CarbonFile[] blockRelatedFiles = updateStatusManager
- .getAllBlockRelatedFiles(block.getBlockName(), allSegmentFiles,
+ .getAllBlockRelatedFiles(allSegmentFiles,
block.getActualBlockName());
// now for each invalid index file need to check the query execution time out
@@ -555,7 +543,7 @@ public class CarbonUpdateUtil {
} else {
invalidDeleteDeltaFiles = updateStatusManager
- .getDeleteDeltaInvalidFilesList(segment.getLoadName(), block, false,
+ .getDeleteDeltaInvalidFilesList(block, false,
allSegmentFiles, isInvalidFile);
for (CarbonFile invalidFile : invalidDeleteDeltaFiles) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
index 7affe12..20db937 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockDetails.java
@@ -48,14 +48,6 @@ public class DeleteDeltaBlockDetails implements Serializable {
blockletDetailsMap = new TreeMap<>();
}
- public String getBlockName() {
- return blockName;
- }
-
- public void setBlockName(String blockName) {
- this.blockName = blockName;
- }
-
@Override public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null || !(obj instanceof DeleteDeltaBlockDetails)) return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
index 22785fa..6185134 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
@@ -46,14 +46,6 @@ public class DeleteDeltaBlockletDetails implements Serializable {
return deletedRows.add(row);
}
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
public Integer getPageId() {
return pageId;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
deleted file mode 100644
index 3c0dd4f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/mutate/data/BlockletDeleteDeltaCacheLoader.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.mutate.data;
-
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-
-/**
- * This class is responsible for loading delete delta file cache based on
- * blocklet id of a particular block
- */
-public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf {
- private String blockletID;
- private DataRefNode blockletNode;
- private AbsoluteTableIdentifier absoluteIdentifier;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName());
-
- public BlockletDeleteDeltaCacheLoader(String blockletID, DataRefNode blockletNode,
- AbsoluteTableIdentifier absoluteIdentifier) {
- this.blockletID = blockletID;
- this.blockletNode = blockletNode;
- this.absoluteIdentifier = absoluteIdentifier;
- }
-
- /**
- * This method will load the delete delta cache based on blocklet id of particular block with
- * the help of SegmentUpdateStatusManager.
- */
- public void loadDeleteDeltaFileDataToCache() {
- SegmentUpdateStatusManager segmentUpdateStatusManager =
- new SegmentUpdateStatusManager(absoluteIdentifier);
- Map<Integer, Integer[]> deleteDeltaFileData = null;
- BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null;
- if (null == blockletNode.getDeleteDeltaDataCache()) {
- try {
- deleteDeltaFileData =
- segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
- } catch (Exception e) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- }
- } else {
- deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
- // if already cache is present then validate the cache using timestamp
- String cacheTimeStamp = segmentUpdateStatusManager
- .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp());
- if (null != cacheTimeStamp) {
- try {
- deleteDeltaFileData =
- segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
- } catch (Exception e) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- }
- }
- }
- blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
index b14660c..88b3065 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
@@ -18,7 +18,7 @@ package org.apache.carbondata.core.scan.collector;
import java.util.List;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
/**
@@ -27,17 +27,13 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
public interface ScannedResultCollector {
/**
- * Below method will be used to aggregate the scanned result
- *
- * @param scannedResult scanned result
- * @return how many records was aggregated
+ * Return the aggregated scanned result in list of rows
*/
- List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
+ List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize);
/**
- * Collects data in columnar format.
- * @param scannedResult
- * @param columnarBatch
+ * Collects result and set it in the specified columnar batch
*/
- void collectVectorBatch(AbstractScannedResult scannedResult, CarbonColumnarBatch columnarBatch);
+ void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
+ CarbonColumnarBatch columnarBatch);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index dc78ac6..694271e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.scan.collector.impl;
import java.math.BigDecimal;
import java.math.RoundingMode;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -29,8 +27,8 @@ import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -39,38 +37,35 @@ import org.apache.carbondata.core.util.DataTypeUtil;
*/
public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
-
/**
* table block execution infos
*/
- protected BlockExecutionInfo tableBlockExecutionInfos;
+ BlockExecutionInfo executionInfo;
/**
* maintains the measure information like datatype, ordinal, measure existence
*/
- protected MeasureInfo measureInfo;
+ MeasureInfo measureInfo;
/**
* maintains the dimension information like datatype, ordinal, measure existence
*/
- protected DimensionInfo dimensionInfo;
+ DimensionInfo dimensionInfo;
- public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- this.tableBlockExecutionInfos = blockExecutionInfos;
+ AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ this.executionInfo = blockExecutionInfos;
measureInfo = blockExecutionInfos.getMeasureInfo();
dimensionInfo = blockExecutionInfos.getDimensionInfo();
}
protected void fillMeasureData(Object[] msrValues, int offset,
- AbstractScannedResult scannedResult) {
+ BlockletScannedResult scannedResult) {
int measureExistIndex = 0;
for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) {
// if measure exists is block then pass measure column
// data chunk to the collector
if (measureInfo.getMeasureExists()[i]) {
- QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex];
+ ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex];
msrValues[i + offset] = getMeasureData(
scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]),
scannedResult.getCurrentRowId(), queryMeasure.getMeasure());
@@ -87,8 +82,7 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
}
}
- protected Object getMeasureData(ColumnPage dataChunk, int index,
- CarbonMeasure carbonMeasure) {
+ Object getMeasureData(ColumnPage dataChunk, int index, CarbonMeasure carbonMeasure) {
if (!dataChunk.getNullBits().get(index)) {
DataType dataType = carbonMeasure.getDataType();
if (dataType == DataTypes.BOOLEAN) {
@@ -114,7 +108,8 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
return null;
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
throw new UnsupportedOperationException("Works only for batch collectors");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index cf6e6d6..4322034 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -29,9 +29,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -42,49 +42,50 @@ import org.apache.commons.lang3.ArrayUtils;
*/
public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
- protected QueryDimension[] queryDimensions;
+ protected ProjectionDimension[] queryDimensions;
- protected QueryMeasure[] queryMeasures;
+ protected ProjectionMeasure[] queryMeasures;
- protected DirectDictionaryGenerator[] directDictionaryGenerators;
+ private DirectDictionaryGenerator[] directDictionaryGenerators;
/**
* query order
*/
protected int[] order;
- protected int[] actualIndexInSurrogateKey;
+ private int[] actualIndexInSurrogateKey;
- protected boolean[] dictionaryEncodingArray;
+ boolean[] dictionaryEncodingArray;
- protected boolean[] directDictionaryEncodingArray;
+ boolean[] directDictionaryEncodingArray;
- protected boolean[] implictColumnArray;
+ private boolean[] implictColumnArray;
- protected boolean[] complexDataTypeArray;
+ private boolean[] complexDataTypeArray;
- protected int dictionaryColumnIndex;
- protected int noDictionaryColumnIndex;
- protected int complexTypeColumnIndex;
+ int dictionaryColumnIndex;
+ int noDictionaryColumnIndex;
+ int complexTypeColumnIndex;
- protected boolean isDimensionExists;
+ boolean isDimensionExists;
- protected Map<Integer, GenericQueryType> comlexDimensionInfoMap;
+ private Map<Integer, GenericQueryType> comlexDimensionInfoMap;
public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ queryDimensions = executionInfo.getProjectionDimensions();
+ queryMeasures = executionInfo.getProjectionMeasures();
initDimensionAndMeasureIndexesForFillingData();
isDimensionExists = queryDimensions.length > 0;
- this.comlexDimensionInfoMap = tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ this.comlexDimensionInfoMap = executionInfo.getComlexDimensionInfoMap();
}
/**
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
// scan the record and add to list
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
@@ -118,13 +119,13 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
return listBasedResult;
}
- protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
+ void fillDimensionData(BlockletScannedResult scannedResult, int[] surrogateResult,
byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
if (!dictionaryEncodingArray[i]) {
if (implictColumnArray[i]) {
if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
- .equals(queryDimensions[i].getDimension().getColName())) {
+ .equals(queryDimensions[i].getColumnName())) {
row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult
.getCurrentPageCounter() + CarbonCommonConstants.FILE_SEPARATOR + scannedResult
@@ -153,7 +154,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
}
- protected void fillMeasureData(AbstractScannedResult scannedResult, Object[] row) {
+ void fillMeasureData(BlockletScannedResult scannedResult, Object[] row) {
if (measureInfo.getMeasureDataTypes().length > 0) {
Object[] msrValues = new Object[measureInfo.getMeasureDataTypes().length];
fillMeasureData(msrValues, 0, scannedResult);
@@ -163,7 +164,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
}
- protected void initDimensionAndMeasureIndexesForFillingData() {
+ void initDimensionAndMeasureIndexesForFillingData() {
List<Integer> dictionaryIndexes = new ArrayList<Integer>();
for (int i = 0; i < queryDimensions.length; i++) {
if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) || queryDimensions[i]
@@ -190,10 +191,10 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
order = new int[queryDimensions.length + queryMeasures.length];
for (int i = 0; i < queryDimensions.length; i++) {
- order[i] = queryDimensions[i].getQueryOrder();
+ order[i] = queryDimensions[i].getOrdinal();
}
for (int i = 0; i < queryMeasures.length; i++) {
- order[i + queryDimensions.length] = queryMeasures[i].getQueryOrder();
+ order[i + queryDimensions.length] = queryMeasures[i].getOrdinal();
}
directDictionaryGenerators = new DirectDictionaryGenerator[queryDimensions.length];
for (int i = 0; i < queryDimensions.length; i++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index e9d6740..4947621 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor;
@@ -35,35 +35,35 @@ import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor;
*/
public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector {
- protected QueryDimension[] queryDimensions;
+ protected ProjectionDimension[] queryDimensions;
- protected QueryMeasure[] queryMeasures;
+ protected ProjectionMeasure[] queryMeasures;
- protected ColumnVectorInfo[] dictionaryInfo;
+ private ColumnVectorInfo[] dictionaryInfo;
- protected ColumnVectorInfo[] noDictionaryInfo;
+ private ColumnVectorInfo[] noDictionaryInfo;
- protected ColumnVectorInfo[] complexInfo;
+ private ColumnVectorInfo[] complexInfo;
- protected ColumnVectorInfo[] measureColumnInfo;
+ private ColumnVectorInfo[] measureColumnInfo;
- protected ColumnVectorInfo[] allColumnInfo;
+ ColumnVectorInfo[] allColumnInfo;
- protected ColumnVectorInfo[] implictColumnInfo;
+ private ColumnVectorInfo[] implictColumnInfo;
public DictionaryBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
// initialize only if the current block is not a restructured block else the initialization
// will be taken care by RestructureBasedVectorResultCollector
if (!blockExecutionInfos.isRestructuredBlock()) {
- queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ queryDimensions = executionInfo.getProjectionDimensions();
+ queryMeasures = executionInfo.getProjectionMeasures();
allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
prepareDimensionAndMeasureColumnVectors();
}
}
- protected void prepareDimensionAndMeasureColumnVectors() {
+ void prepareDimensionAndMeasureColumnVectors() {
measureColumnInfo = new ColumnVectorInfo[queryMeasures.length];
List<ColumnVectorInfo> dictInfoList = new ArrayList<>();
List<ColumnVectorInfo> noDictInfoList = new ArrayList<>();
@@ -78,13 +78,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
implictColumnList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
noDictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
dictInfoList.add(columnVectorInfo);
@@ -92,21 +92,21 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else if (queryDimensions[i].getDimension().isComplex()) {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
complexList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
columnVectorInfo.genericQueryType =
- tableBlockExecutionInfos.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ executionInfo.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
} else {
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
dictInfoList.add(columnVectorInfo);
columnVectorInfo.dimension = queryDimensions[i];
columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
}
}
for (int i = 0; i < queryMeasures.length; i++) {
@@ -116,7 +116,7 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnVectorInfo.ordinal = queryMeasures[i].getMeasure().getOrdinal();
columnVectorInfo.measure = queryMeasures[i];
this.measureColumnInfo[i] = columnVectorInfo;
- allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo;
}
dictionaryInfo = dictInfoList.toArray(new ColumnVectorInfo[dictInfoList.size()]);
noDictionaryInfo = noDictInfoList.toArray(new ColumnVectorInfo[noDictInfoList.size()]);
@@ -126,11 +126,13 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
Arrays.sort(complexInfo);
}
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- throw new UnsupportedOperationException("collectData is not supported here");
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
+ throw new UnsupportedOperationException("collectResultInRow is not supported here");
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
int numberOfPages = scannedResult.numberOfpages();
int filteredRows = 0;
@@ -150,14 +152,15 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
return;
}
fillColumnVectorDetails(columnarBatch, rowCounter, requiredRows);
- filteredRows = scannedResult
- .markFilteredRows(columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter());
- scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
+ filteredRows = scannedResult.markFilteredRows(
+ columnarBatch, rowCounter, requiredRows, columnarBatch.getRowCounter());
+ fillResultToColumnarBatch(
+ scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows);
}
}
- protected void scanAndFillResult(AbstractScannedResult scannedResult,
+ void fillResultToColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch, int rowCounter, int availableRows, int requiredRows) {
scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
@@ -174,8 +177,8 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
}
- protected void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter,
- int requiredRows) {
+ void fillColumnVectorDetails(CarbonColumnarBatch columnarBatch, int rowCounter, int requiredRows)
+ {
for (int i = 0; i < allColumnInfo.length; i++) {
allColumnInfo[i].size = requiredRows;
allColumnInfo[i].offset = rowCounter;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
index a09676a..0780675 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -16,14 +16,12 @@
*/
package org.apache.carbondata.core.scan.collector.impl;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
@@ -31,15 +29,11 @@ import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
*/
public class RawBasedResultCollector extends AbstractScannedResultCollector {
- protected ByteArrayWrapper wrapper;
+ byte[] dictionaryKeyArray;
- protected byte[] dictionaryKeyArray;
+ byte[][] noDictionaryKeyArray;
- protected byte[][] noDictionaryKeyArray;
-
- protected byte[][] complexTypeKeyArray;
-
- protected byte[] implicitColumnByteArray;
+ private byte[][] complexTypeKeyArray;
public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
@@ -49,9 +43,10 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ ProjectionMeasure[] queryMeasures = executionInfo.getProjectionMeasures();
// scan the record and add to list
int rowCounter = 0;
while (scannedResult.hasNext() && rowCounter < batchSize) {
@@ -65,24 +60,21 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
return listBasedResult;
}
- protected void prepareRow(AbstractScannedResult scannedResult, List<Object[]> listBasedResult,
- QueryMeasure[] queryMeasures) {
+ void prepareRow(BlockletScannedResult scannedResult, List<Object[]> listBasedResult,
+ ProjectionMeasure[] queryMeasures) {
Object[] row = new Object[1 + queryMeasures.length];
- wrapper = new ByteArrayWrapper();
+ ByteArrayWrapper wrapper = new ByteArrayWrapper();
wrapper.setDictionaryKey(dictionaryKeyArray);
wrapper.setNoDictionaryKeys(noDictionaryKeyArray);
wrapper.setComplexTypesKeys(complexTypeKeyArray);
- wrapper.setImplicitColumnByteArray(implicitColumnByteArray);
row[0] = wrapper;
fillMeasureData(row, 1, scannedResult);
listBasedResult.add(row);
}
- protected void scanResultAndGetData(AbstractScannedResult scannedResult) {
+ void scanResultAndGetData(BlockletScannedResult scannedResult) {
dictionaryKeyArray = scannedResult.getDictionaryKeyArray();
noDictionaryKeyArray = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
- implicitColumnByteArray = scannedResult.getBlockletId()
- .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 296d9a6..8b42a4a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -23,8 +23,8 @@ import java.util.Map;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
/**
* class for handling restructure scenarios for filling result
@@ -35,8 +35,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
public RestructureBasedDictionaryResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ queryDimensions = executionInfo.getActualQueryDimensions();
+ queryMeasures = executionInfo.getActualQueryMeasures();
measureDefaultValues = new Object[queryMeasures.length];
fillMeasureDefaultValues();
initDimensionAndMeasureIndexesForFillingData();
@@ -61,7 +61,8 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
// scan the record and add to list
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
@@ -69,7 +70,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
Map<Integer, GenericQueryType> comlexDimensionInfoMap =
- tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ executionInfo.getComlexDimensionInfoMap();
while (scannedResult.hasNext() && rowCounter < batchSize) {
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
@@ -107,13 +108,13 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
}
protected void fillMeasureData(Object[] msrValues, int offset,
- AbstractScannedResult scannedResult) {
+ BlockletScannedResult scannedResult) {
int measureExistIndex = 0;
for (short i = 0; i < measureInfo.getMeasureDataTypes().length; i++) {
// if measure exists is block then pass measure column
// data chunk to the collector
if (measureInfo.getMeasureExists()[i]) {
- QueryMeasure queryMeasure = tableBlockExecutionInfos.getQueryMeasures()[measureExistIndex];
+ ProjectionMeasure queryMeasure = executionInfo.getProjectionMeasures()[measureExistIndex];
msrValues[i + offset] = getMeasureData(
scannedResult.getMeasureChunk(measureInfo.getMeasureOrdinals()[measureExistIndex]),
scannedResult.getCurrentRowId(), queryMeasure.getMeasure());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
index b08a1d4..6544a75 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.lang3.ArrayUtils;
@@ -70,15 +70,15 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
*/
private void initRestructuredKeyGenerator() {
SegmentProperties segmentProperties =
- tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
- QueryDimension[] queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ executionInfo.getDataBlock().getSegmentProperties();
+ ProjectionDimension[] queryDimensions = executionInfo.getActualQueryDimensions();
List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
- int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
+ int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
int dimCounterInCurrentBlock = 0;
for (int i = 0; i < queryDimensions.length; i++) {
if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
- if (tableBlockExecutionInfos.getDimensionInfo().getDimensionExists()[i]) {
+ if (executionInfo.getDimensionInfo().getDimensionExists()[i]) {
// get the dictionary key ordinal as column cardinality in segment properties
// will only be for dictionary encoded columns
CarbonDimension currentBlockDimension = segmentProperties.getDimensions()
@@ -124,8 +124,8 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
*/
private void initCurrentBlockKeyGenerator() {
SegmentProperties segmentProperties =
- tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
- int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
+ executionInfo.getDataBlock().getSegmentProperties();
+ int[] dictionaryColumnBlockIndex = executionInfo.getDictionaryColumnChunkIndex();
int[] updatedColumnCardinality = new int[dictionaryColumnBlockIndex.length];
int[] updatedDimensionPartitioner = new int[dictionaryColumnBlockIndex.length];
for (int i = 0; i < dictionaryColumnBlockIndex.length; i++) {
@@ -149,9 +149,10 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* This method will add a record both key and value to list object
* it will keep track of how many record is processed, to handle limit scenario
*/
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ ProjectionMeasure[] queryMeasures = executionInfo.getActualQueryMeasures();
// scan the record and add to list
int rowCounter = 0;
while (scannedResult.hasNext() && rowCounter < batchSize) {
@@ -179,7 +180,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* @return
*/
private byte[] fillDictionaryKeyArrayWithLatestSchema(byte[] dictionaryKeyArray) {
- QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions();
int newKeyArrayLength = dimensionInfo.getNewDictionaryColumnCount();
long[] keyArray = null;
if (null != updatedCurrentBlockKeyGenerator) {
@@ -222,7 +223,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
* @return
*/
private byte[][] fillNoDictionaryKeyArrayWithLatestSchema(byte[][] noDictionaryKeyArray) {
- QueryDimension[] actualQueryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
+ ProjectionDimension[] actualQueryDimensions = executionInfo.getActualQueryDimensions();
byte[][] noDictionaryKeyArrayWithNewlyAddedColumns =
new byte[noDictionaryKeyArray.length + dimensionInfo.getNewNoDictionaryColumnCount()][];
int existingColumnValueIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
index 0f1b9f9..61a2992 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedVectorResultCollector.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
-import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.BlockletScannedResult;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -43,8 +43,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
public RestructureBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
super(blockExecutionInfos);
- queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
- queryMeasures = tableBlockExecutionInfos.getActualQueryMeasures();
+ queryDimensions = executionInfo.getActualQueryDimensions();
+ queryMeasures = executionInfo.getActualQueryMeasures();
measureDefaultValues = new Object[queryMeasures.length];
allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
createVectorForNewlyAddedDimensions();
@@ -66,7 +66,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
}
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryDimensions[i].getOrdinal()] = columnVectorInfo;
}
}
}
@@ -79,7 +79,7 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
if (!measureInfo.getMeasureExists()[i]) {
// add a dummy column vector result collector object
ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+ allColumnInfo[queryMeasures[i].getOrdinal()] = columnVectorInfo;
columnVectorInfo.measure = queryMeasures[i];
measureDefaultValues[i] = getMeasureDefaultValue(queryMeasures[i].getMeasure());
}
@@ -97,11 +97,13 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
carbonMeasure.getDefaultValue());
}
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- throw new UnsupportedOperationException("collectData is not supported here");
+ @Override
+ public List<Object[]> collectResultInRow(BlockletScannedResult scannedResult, int batchSize) {
+ throw new UnsupportedOperationException("collectResultInRow is not supported here");
}
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ @Override
+ public void collectResultInColumnarBatch(BlockletScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
int numberOfPages = scannedResult.numberOfpages();
while (scannedResult.getCurrentPageCounter() < numberOfPages) {
@@ -124,7 +126,8 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
fillDataForNonExistingDimensions();
fillDataForNonExistingMeasures();
// fill existing dimensions and measures data
- scanAndFillResult(scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
+ fillResultToColumnarBatch(
+ scannedResult, columnarBatch, rowCounter, availableRows, requiredRows);
columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows - filteredRows);
}
}
@@ -133,11 +136,11 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
* This method will fill the default values of non existing dimensions in the current block
*/
private void fillDataForNonExistingDimensions() {
- for (int i = 0; i < tableBlockExecutionInfos.getActualQueryDimensions().length; i++) {
+ for (int i = 0; i < executionInfo.getActualQueryDimensions().length; i++) {
if (!dimensionInfo.getDimensionExists()[i]) {
- int queryOrder = tableBlockExecutionInfos.getActualQueryDimensions()[i].getQueryOrder();
+ int queryOrder = executionInfo.getActualQueryDimensions()[i].getOrdinal();
CarbonDimension dimension =
- tableBlockExecutionInfos.getActualQueryDimensions()[i].getDimension();
+ executionInfo.getActualQueryDimensions()[i].getDimension();
if (dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
// fill direct dictionary column data
fillDirectDictionaryData(allColumnInfo[queryOrder].vector, allColumnInfo[queryOrder],
@@ -215,10 +218,10 @@ public class RestructureBasedVectorResultCollector extends DictionaryBasedVector
* This method will fill the default values of non existing measures in the current block
*/
private void fillDataForNonExistingMeasures() {
- for (int i = 0; i < tableBlockExecutionInfos.getActualQueryMeasures().length; i++) {
+ for (int i = 0; i < executionInfo.getActualQueryMeasures().length; i++) {
if (!measureInfo.getMeasureExists()[i]) {
- int queryOrder = tableBlockExecutionInfos.getActualQueryMeasures()[i].getQueryOrder();
- CarbonMeasure measure = tableBlockExecutionInfos.getActualQueryMeasures()[i].getMeasure();
+ int queryOrder = executionInfo.getActualQueryMeasures()[i].getOrdinal();
+ CarbonMeasure measure = executionInfo.getActualQueryMeasures()[i].getMeasure();
ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder];
CarbonColumnVector vector = columnVectorInfo.vector;
Object defaultValue = measureDefaultValues[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
index 1fcccca..30dd1dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
@@ -23,7 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.catalyst.util.GenericArrayData;
import org.apache.spark.sql.types.ArrayType;
@@ -86,7 +86,7 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
return new ArrayType(null, true);
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
children.fillRequiredBlockData(blockChunkHolder);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
index ee43a10..98f0715 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.scan.complextypes;
import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
public class ComplexQueryType {
protected String name;
@@ -42,7 +42,7 @@ public class ComplexQueryType {
protected byte[] copyBlockDataChunk(DimensionRawColumnChunk[] rawColumnChunks,
int rowNumber, int pageNumber) {
byte[] data =
- rawColumnChunks[blockIndex].convertToDimColDataChunk(pageNumber).getChunkData(rowNumber);
+ rawColumnChunks[blockIndex].decodeColumnPage(pageNumber).getChunkData(rowNumber);
byte[] output = new byte[data.length];
System.arraycopy(data, 0, output, 0, output.length);
return output;
@@ -51,10 +51,10 @@ public class ComplexQueryType {
/*
* This method will read the block data chunk from the respective block
*/
- protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ protected void readBlockDataChunk(RawBlockletColumnChunks blockChunkHolder) throws IOException {
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[blockIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[blockIndex] = blockChunkHolder.getDataBlock()
+ .readDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index 978a652..b8aa912 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.Bits;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.spark.sql.types.BooleanType$;
@@ -113,7 +113,7 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
}
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
index 23a9f81..1d4f141 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.types.DataType;
@@ -106,7 +106,7 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
return new StructType(fields);
}
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ @Override public void fillRequiredBlockData(RawBlockletColumnChunks blockChunkHolder)
throws IOException {
readBlockDataChunk(blockChunkHolder);
[02/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index f4450e3..5f8d199 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
import mockit.Mock;
import mockit.MockUp;
@@ -267,8 +267,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, 1);
}
@@ -276,8 +276,8 @@ public class CarbonUtilTest {
@Test public void testToGetNextLesserValueToTarget() {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, -1);
}
@@ -285,8 +285,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValue() {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 3);
}
@@ -302,8 +302,8 @@ public class CarbonUtilTest {
@Test public void testToGetnextGreaterValueToTarget() {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
- FixedLengthDimensionDataChunk fixedLengthDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 5, 1);
+ FixedLengthDimensionColumnPage fixedLengthDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 5);
}
@@ -525,23 +525,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDictionaryEncodingArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List<Encoding> encoding = new ArrayList<>();
encoding.add(Encoding.DICTIONARY);
column1Schema.setEncodingList(encoding);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { true, false };
@@ -551,23 +551,23 @@ public class CarbonUtilTest {
}
@Test public void testToGetDirectDictionaryEncodingArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
List<Encoding> encoding = new ArrayList<>();
encoding.add(Encoding.DIRECT_DICTIONARY);
column1Schema.setEncodingList(encoding);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
List<Encoding> encoding2 = new ArrayList<>();
encoding2.add(Encoding.DELTA);
column2Schema.setEncodingList(encoding2);
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { true, false };
@@ -577,19 +577,19 @@ public class CarbonUtilTest {
}
@Test public void testToGetComplexDataTypeArray() {
- QueryDimension column1 = new QueryDimension("Column1");
- QueryDimension column2 = new QueryDimension("Column2");
ColumnSchema column1Schema = new ColumnSchema();
ColumnSchema column2Schema = new ColumnSchema();
column1Schema.setColumnName("Column1");
column1Schema.setDataType(DataTypes.DATE);
- column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
column2Schema.setDataType(DataTypes.createDefaultArrayType());
- column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
+ ProjectionDimension
+ column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
- QueryDimension[] queryDimensions = { column1, column2 };
+ ProjectionDimension[] queryDimensions = { column1, column2 };
boolean[] dictionaryEncoding = CarbonUtil.getComplexDataTypeArray(queryDimensions);
boolean[] expectedDictionaryEncoding = { false, true };
@@ -806,8 +806,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo1() {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 5 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(-2, result);
@@ -816,8 +816,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareToLessThan0() {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 30 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(2, result);
@@ -826,8 +826,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithCompareTo0() {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(0, result);
@@ -836,8 +836,8 @@ public class CarbonUtilTest {
@Test public void testToGetFirstIndexUsingBinarySearchWithMatchUpLimitTrue() {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk =
- new FixedLengthDimensionDataChunk(dataChunks, null, null, 6, 1);
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true);
assertEquals(2, result);
@@ -847,13 +847,13 @@ public class CarbonUtilTest {
public void testBinaryRangeSearch() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[1];
int[] range;
dataChunk = "abbcccddddeffgggh".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -885,7 +885,7 @@ public class CarbonUtilTest {
assertRangeIndex(dataArr, dataChunk, fixedLengthDimensionDataChunk, keyWord, expectRangeIndex);
dataChunk = "ab".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -899,7 +899,7 @@ public class CarbonUtilTest {
assertEquals(1, range[1]);
dataChunk = "aabb".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -913,7 +913,7 @@ public class CarbonUtilTest {
assertEquals(3, range[1]);
dataChunk = "a".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -922,7 +922,7 @@ public class CarbonUtilTest {
assertEquals(0, range[1]);
dataChunk = "aa".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("97");
@@ -931,7 +931,7 @@ public class CarbonUtilTest {
assertEquals(1, range[1]);
dataChunk = "aabbbbbbbbbbcc".getBytes();
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
keyWord[0] = Byte.valueOf("98");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -944,14 +944,14 @@ public class CarbonUtilTest {
public void IndexUsingBinarySearchLengthTwo() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[2];
dataChunk = "aabbbbbbbbbbcc".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -986,14 +986,14 @@ public class CarbonUtilTest {
public void IndexUsingBinarySearchLengthThree() {
byte[] dataChunk = new byte[10];
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk;
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk;
byte[] keyWord = new byte[3];
dataChunk = "aaabbbbbbbbbccc".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
- fixedLengthDimensionDataChunk = new FixedLengthDimensionDataChunk(dataChunk, null, null,
+ fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
dataChunk.length / keyWord.length, keyWord.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
@@ -1101,7 +1101,7 @@ public class CarbonUtilTest {
}
private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
- FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
+ FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
int[] range;
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0,
(dataChunk.length - 1) / keyWord.length, keyWord);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index de64c0a..e506994 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -25,10 +25,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
+import org.apache.carbondata.core.datastore.impl.FileReaderImpl;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
@@ -229,13 +229,13 @@ public class DataFileFooterConverterTest {
}
@SuppressWarnings("unused") @Mock
- public FileHolder getFileHolder(FileFactory.FileType fileType) {
- return new FileHolderImpl();
+ public FileReader getFileHolder(FileFactory.FileType fileType) {
+ return new FileReaderImpl();
}
};
- new MockUp<FileHolderImpl>() {
+ new MockUp<FileReaderImpl>() {
@SuppressWarnings("unused") @Mock public long readLong(String filePath, long offset) {
return 1;
}
@@ -249,7 +249,6 @@ public class DataFileFooterConverterTest {
SegmentInfo segmentInfo = new SegmentInfo();
int[] arr = { 1, 2, 3 };
segmentInfo.setColumnCardinality(arr);
- segmentInfo.setNumberOfColumns(segmentInfo1.getNum_cols());
dataFileFooter.setNumberOfRows(3);
dataFileFooter.setSegmentInfo(segmentInfo);
TableBlockInfo info = new TableBlockInfo("/file.carbondata", 1, "0", new String[0], 1, ColumnarFormatVersion.V1, null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 4c9a784..4fb5dcc 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
import org.apache.carbondata.core.scan.filter.executer.RangeValueFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
import mockit.Deencapsulation;
@@ -102,7 +101,7 @@ public class RangeFilterProcessorTest {
new LessThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING),
new LiteralExpression("20", DataTypes.STRING))), new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, output);
Assert.assertTrue(result);
@@ -143,7 +142,7 @@ public class RangeFilterProcessorTest {
new LessThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING),
new LiteralExpression("05", DataTypes.STRING)));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, output);
// no change
@@ -218,7 +217,7 @@ public class RangeFilterProcessorTest {
Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, new AndExpression(Andb3, new TrueExpression(null)));
// no change
@@ -302,7 +301,7 @@ public class RangeFilterProcessorTest {
Expression Orb3 = new OrExpression(Orb2, lessThanb2);
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ new RangeFilterOptmizer(inputFilter);
rangeFilterOptimizer.optimizeFilter();
result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
// no change
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java b/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
deleted file mode 100644
index 94c3f68..0000000
--- a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.scanner.impl;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class FilterScannerTest {
-//
-// private static FilterScanner filterScanner;
-// private static BlockletIndex blockletIndex;
-// private static BlockletMinMaxIndex blockletMinMaxIndex;
-// private static BTreeBuilderInfo bTreeBuilderInfo;
-// private static DataFileFooter dataFileFooter;
-//
-// @BeforeClass public static void setUp() {
-// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
-// FilterExecuter filterExecutor = new AndFilterExecuterImpl(null, null);
-// blockExecutionInfo.setFilterExecuterTree(filterExecutor);
-// blockExecutionInfo.setFixedLengthKeySize(1);
-// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 1, 2 });
-// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 1 });
-// blockExecutionInfo.setColumnGroupToKeyStructureInfo(new HashMap<Integer, KeyStructureInfo>());
-// blockExecutionInfo.setComplexDimensionInfoMap(new HashMap<Integer, GenericQueryType>());
-// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
-// blockExecutionInfo.setQueryDimensions(new QueryDimension[] { new QueryDimension("Col1") });
-// blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setTotalNumberOfMeasureBlock(1);
-// blockExecutionInfo.setTotalNumberDimensionBlock(1);
-// QueryStatisticsModel queryStatisticsModel = new QueryStatisticsModel();
-// QueryStatistic queryStatistic = new QueryStatistic();
-// queryStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 1);
-// Map<String, QueryStatistic> statisticsTypeAndObjMap = new HashMap<>();
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatistic);
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatistic);
-// queryStatisticsModel.setStatisticsTypeAndObjMap(statisticsTypeAndObjMap);
-// QueryStatisticsRecorder queryStatisticsRecorder = new QueryStatisticsRecorderImpl("1");
-// queryStatisticsModel.setRecorder(queryStatisticsRecorder);
-// filterScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
-// blockletIndex = new BlockletIndex();
-// blockletMinMaxIndex = new BlockletMinMaxIndex();
-// blockletMinMaxIndex.setMinValues(new byte[][] { { 1, 2 } });
-// blockletMinMaxIndex.setMaxValues(new byte[][] { { 10, 12 } });
-// blockletIndex.setMinMaxIndex(blockletMinMaxIndex);
-// dataFileFooter = new DataFileFooter();
-// dataFileFooter.setBlockletIndex(blockletIndex);
-// bTreeBuilderInfo = new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// }
-//
-// @Test public void testToScanBlockletWithEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// return new BitSet();
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-// assertEquals(0, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test public void testToScanBlockletWithNonEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-// };
-// DataRefNode dataRefNode = new MockUp<DataRefNode>() {
-// @Mock @SuppressWarnings("unused") DimensionColumnDataChunk[] getDimensionChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-// DimensionColumnDataChunk[] dimensionChunkAttributes =
-// { new ColumnGroupDimensionDataChunk(null, null) };
-// return dimensionChunkAttributes;
-// }
-//
-// @Mock @SuppressWarnings("unused") ColumnPage[] getMeasureChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-//
-// ColumnPage[] ColumnPages = { new ColumnPage() };
-// return ColumnPages;
-// }
-// }.getMockInstance();
-//
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
-// DimensionColumnDataChunk dimensionColumnDataChunk =
-// new FixedLengthDimensionDataChunk(new byte[] { 0, 1 }, dimensionChunkAttributes);
-// blocksChunkHolder.setDimensionRawDataChunk(new DimensionColumnDataChunk[]
-//
-// { dimensionColumnDataChunk });
-// ColumnPage ColumnPage = new ColumnPage();
-// blocksChunkHolder.setMeasureDataChunk(new ColumnPage[]
-//
-// { ColumnPage });
-// FileHolder fileHolder = new DFSFileHolderImpl();
-// blocksChunkHolder.setFileReader(fileHolder);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-//
-// assertEquals(2, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test(expected = QueryExecutionException.class) public void testToScanBlockletWithException()
-// throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// throw new FilterUnsupportedException("Filter unsupported");
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// BTreeBuilderInfo bTreeBuilderInfo =
-// new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// filterScanner.scanBlocklet(blocksChunkHolder);
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/dev/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/dev/findbugs-exclude.xml b/dev/findbugs-exclude.xml
index 1520cd4..b19db85 100644
--- a/dev/findbugs-exclude.xml
+++ b/dev/findbugs-exclude.xml
@@ -31,7 +31,7 @@
<Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
<Match>
- <Class name="org.apache.carbondata.core.datastore.impl.FileHolderImpl"/>
+ <Class name="org.apache.carbondata.core.datastore.impl.FileReaderImpl"/>
<Method name="getDataInputStream"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION_EXCEPTION_EDGE"/>
</Match>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 76afcbf..7a15327 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -62,7 +62,7 @@ object CarbonSessionExample {
spark.sql(
s"""
- | SELECT *
+ | SELECT charField, stringField, intField
| FROM carbon_table
| WHERE stringfield = 'spark' AND decimalField > 40
""".stripMargin).show()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index fdec267..5a20d7e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -58,7 +58,6 @@ import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
@@ -112,11 +111,11 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// comma separated list of input segment numbers
public static final String INPUT_SEGMENT_NUMBERS =
"mapreduce.input.carboninputformat.segmentnumbers";
- public static final String VALIDATE_INPUT_SEGMENT_IDs =
+ private static final String VALIDATE_INPUT_SEGMENT_IDs =
"mapreduce.input.carboninputformat.validsegments";
// comma separated list of input files
public static final String INPUT_FILES = "mapreduce.input.carboninputformat.files";
- public static final String ALTER_PARTITION_ID = "mapreduce.input.carboninputformat.partitionid";
+ private static final String ALTER_PARTITION_ID = "mapreduce.input.carboninputformat.partitionid";
private static final Log LOG = LogFactory.getLog(CarbonTableInputFormat.class);
private static final String FILTER_PREDICATE =
"mapreduce.input.carboninputformat.filter.predicate";
@@ -127,7 +126,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
private static final String DATA_MAP_DSTR = "mapreduce.input.carboninputformat.datamapdstr";
public static final String DATABASE_NAME = "mapreduce.input.carboninputformat.databaseName";
public static final String TABLE_NAME = "mapreduce.input.carboninputformat.tableName";
- public static final String PARTITIONS_TO_PRUNE =
+ private static final String PARTITIONS_TO_PRUNE =
"mapreduce.input.carboninputformat.partitions.to.prune";
public static final String UPADTE_T =
"mapreduce.input.carboninputformat.partitions.to.prune";
@@ -340,7 +339,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
* @return List<InputSplit> list of CarbonInputSplit
* @throws IOException
*/
- @Override public List<InputSplit> getSplits(JobContext job) throws IOException {
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager
.readTableStatusFile(CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()));
@@ -811,28 +811,29 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return split;
}
- @Override public RecordReader<Void, T> createRecordReader(InputSplit inputSplit,
+ @Override
+ public RecordReader<Void, T> createRecordReader(InputSplit inputSplit,
TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
Configuration configuration = taskAttemptContext.getConfiguration();
- QueryModel queryModel = getQueryModel(inputSplit, taskAttemptContext);
+ QueryModel queryModel = createQueryModel(inputSplit, taskAttemptContext);
CarbonReadSupport<T> readSupport = getReadSupportClass(configuration);
return new CarbonRecordReader<T>(queryModel, readSupport);
}
- public QueryModel getQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ public QueryModel createQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException {
Configuration configuration = taskAttemptContext.getConfiguration();
CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
TableProvider tableProvider = new SingleTableProvider(carbonTable);
- // getting the table absoluteTableIdentifier from the carbonTable
- // to avoid unnecessary deserialization
- AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
// query plan includes projection column
- String projection = getColumnProjection(configuration);
- CarbonQueryPlan queryPlan = CarbonInputFormatUtil.createQueryPlan(carbonTable, projection);
- QueryModel queryModel = QueryModel.createModel(identifier, queryPlan, carbonTable,
- getDataTypeConverter(configuration));
+ String projectionString = getColumnProjection(configuration);
+ String[] projectionColumnNames = null;
+ if (projectionString != null) {
+ projectionColumnNames = projectionString.split(",");
+ }
+ QueryModel queryModel = carbonTable.createQueryWithProjection(
+ projectionColumnNames, getDataTypeConverter(configuration));
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
@@ -887,7 +888,8 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
return readSupport;
}
- @Override protected boolean isSplitable(JobContext context, Path filename) {
+ @Override
+ protected boolean isSplitable(JobContext context, Path filename) {
try {
// Don't split the file if it is local file system
FileSystem fileSystem = filename.getFileSystem(context.getConfiguration());
@@ -901,16 +903,6 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
}
/**
- * required to be moved to core
- *
- * @return updateExtension
- */
- private String getUpdateExtension() {
- // TODO: required to modify when supporting update, mostly will be update timestamp
- return "update";
- }
-
- /**
* return valid segment to access
*/
public Segment[] getSegmentsToAccess(JobContext job) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index a590a5b..0fe0cbf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -176,9 +176,7 @@ class InMemoryBTreeIndex implements Index {
filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
abstractIndex.getDataRefNode(),
resolver,
- abstractIndex,
- identifier
- );
+ abstractIndex);
}
resultFilterredBlocks.addAll(filterredBlocks);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 95a7af0..1e227c4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -156,7 +156,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
hadoopConf = context.getConfiguration();
if (model == null) {
CarbonTableInputFormat format = new CarbonTableInputFormat<Object>();
- model = format.getQueryModel(split, context);
+ model = format.createQueryModel(split, context);
}
carbonTable = model.getTable();
List<CarbonDimension> dimensions =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index 89a4a9a..2f28861 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -67,7 +67,7 @@ public class BlockLevelTraverser {
blockName = CarbonTablePath.getCarbonDataFileName(blockName);
blockName = blockName + CarbonTablePath.getCarbonDataExtension();
- long rowCount = currentBlock.nodeSize();
+ long rowCount = currentBlock.numRows();
String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 056c27b..9f8c5ec 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -26,18 +26,12 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
-import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
@@ -52,45 +46,14 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
*/
public class CarbonInputFormatUtil {
- public static CarbonQueryPlan createQueryPlan(CarbonTable carbonTable, String columnString) {
- String[] columns = null;
- if (columnString != null) {
- columns = columnString.split(",");
- }
- String factTableName = carbonTable.getTableName();
- CarbonQueryPlan plan = new CarbonQueryPlan(carbonTable.getDatabaseName(), factTableName);
- // fill dimensions
- // If columns are null, set all dimensions and measures
- int i = 0;
- if (columns != null) {
- for (String column : columns) {
- CarbonDimension dimensionByName = carbonTable.getDimensionByName(factTableName, column);
- if (dimensionByName != null) {
- addQueryDimension(plan, i, dimensionByName);
- i++;
- } else {
- CarbonMeasure measure = carbonTable.getMeasureByName(factTableName, column);
- if (measure == null) {
- throw new RuntimeException(column + " column not found in the table " + factTableName);
- }
- addQueryMeasure(plan, i, measure);
- i++;
- }
- }
- }
-
- plan.setQueryId(System.nanoTime() + "");
- return plan;
- }
-
public static <V> CarbonTableInputFormat<V> createCarbonInputFormat(
AbsoluteTableIdentifier identifier,
Job job) throws IOException {
CarbonTableInputFormat<V> carbonInputFormat = new CarbonTableInputFormat<>();
- carbonInputFormat.setDatabaseName(job.getConfiguration(),
- identifier.getCarbonTableIdentifier().getDatabaseName());
- carbonInputFormat
- .setTableName(job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
+ CarbonTableInputFormat.setDatabaseName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getDatabaseName());
+ CarbonTableInputFormat.setTableName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
FileInputFormat.addInputPath(job, new Path(identifier.getTablePath()));
return carbonInputFormat;
}
@@ -98,30 +61,16 @@ public class CarbonInputFormatUtil {
public static <V> CarbonTableInputFormat<V> createCarbonTableInputFormat(
AbsoluteTableIdentifier identifier, List<String> partitionId, Job job) throws IOException {
CarbonTableInputFormat<V> carbonTableInputFormat = new CarbonTableInputFormat<>();
- carbonTableInputFormat.setPartitionIdList(job.getConfiguration(), partitionId);
- carbonTableInputFormat.setDatabaseName(job.getConfiguration(),
- identifier.getCarbonTableIdentifier().getDatabaseName());
- carbonTableInputFormat
- .setTableName(job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
+ CarbonTableInputFormat.setPartitionIdList(
+ job.getConfiguration(), partitionId);
+ CarbonTableInputFormat.setDatabaseName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getDatabaseName());
+ CarbonTableInputFormat.setTableName(
+ job.getConfiguration(), identifier.getCarbonTableIdentifier().getTableName());
FileInputFormat.addInputPath(job, new Path(identifier.getTablePath()));
return carbonTableInputFormat;
}
- private static void addQueryMeasure(CarbonQueryPlan plan, int order, CarbonMeasure measure) {
- QueryMeasure queryMeasure = new QueryMeasure(measure.getColName());
- queryMeasure.setQueryOrder(order);
- queryMeasure.setMeasure(measure);
- plan.addMeasure(queryMeasure);
- }
-
- private static void addQueryDimension(CarbonQueryPlan plan, int order,
- CarbonDimension dimension) {
- QueryDimension queryDimension = new QueryDimension(dimension.getColName());
- queryDimension.setQueryOrder(order);
- queryDimension.setDimension(dimension);
- plan.addDimension(queryDimension);
- }
-
public static void processFilterExpression(Expression filterExpression, CarbonTable carbonTable,
boolean[] isFilterDimensions, boolean[] isFilterMeasures) {
QueryModel.processFilterExpression(carbonTable, filterExpression, isFilterDimensions,
@@ -130,7 +79,7 @@ public class CarbonInputFormatUtil {
if (null != filterExpression) {
// Optimize Filter Expression and fit RANGE filters is conditions apply.
FilterOptimizer rangeFilterOptimizer =
- new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
+ new RangeFilterOptmizer(filterExpression);
rangeFilterOptimizer.optimizeFilter();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
index f109e1c..1b57f93 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
@@ -30,7 +30,6 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.SingleTableProvider;
import org.apache.carbondata.core.scan.filter.TableProvider;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.util.DataTypeConverterImpl;
import org.apache.carbondata.hadoop.CarbonInputSplit;
@@ -140,11 +139,11 @@ public class MapredCarbonInputFormat extends CarbonTableInputFormat<ArrayWritabl
AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
- String projection = getProjection(configuration, carbonTable,
+ String projectionString = getProjection(configuration, carbonTable,
identifier.getCarbonTableIdentifier().getTableName());
- CarbonQueryPlan queryPlan = CarbonInputFormatUtil.createQueryPlan(carbonTable, projection);
- QueryModel queryModel =
- QueryModel.createModel(identifier, queryPlan, carbonTable, new DataTypeConverterImpl());
+ String[] projectionColumns = projectionString.split(",");
+ QueryModel queryModel = carbonTable.createQueryWithProjection(
+ projectionColumns, new DataTypeConverterImpl());
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable, null, null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
deleted file mode 100644
index 9a8f8c5..0000000
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.presto;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.StructField;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.scan.executor.QueryExecutor;
-import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
-import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
-import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.hadoop.AbstractRecordReader;
-import org.apache.carbondata.hadoop.CarbonInputSplit;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-/**
- * A specialized RecordReader that reads into InternalRows or ColumnarBatches directly using the
- * carbondata column APIs and fills the data directly into columns.
- */
-class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
-
- private int batchIdx = 0;
-
- private int numBatched = 0;
-
- private CarbonVectorBatch columnarBatch;
-
- private CarbonColumnarBatch carbonColumnarBatch;
-
- /**
- * If true, this class returns batches instead of rows.
- */
- private boolean returnColumnarBatch;
-
- private QueryModel queryModel;
-
- private AbstractDetailQueryResultIterator iterator;
-
- private QueryExecutor queryExecutor;
-
- public CarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel, AbstractDetailQueryResultIterator iterator) {
- this.queryModel = queryModel;
- this.iterator = iterator;
- this.queryExecutor = queryExecutor;
- enableReturningBatches();
- }
-
- /**
- * Implementation of RecordReader API.
- */
- @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
- throws IOException, InterruptedException, UnsupportedOperationException {
- // The input split can contain single HDFS block or multiple blocks, so firstly get all the
- // blocks and then set them in the query model.
- List<CarbonInputSplit> splitList;
- if (inputSplit instanceof CarbonInputSplit) {
- splitList = new ArrayList<>(1);
- splitList.add((CarbonInputSplit) inputSplit);
- } else if (inputSplit instanceof CarbonMultiBlockSplit) {
- // contains multiple blocks, this is an optimization for concurrent query.
- CarbonMultiBlockSplit multiBlockSplit = (CarbonMultiBlockSplit) inputSplit;
- splitList = multiBlockSplit.getAllSplits();
- } else {
- throw new RuntimeException("unsupported input split type: " + inputSplit);
- }
- List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
- queryModel.setTableBlockInfos(tableBlockInfoList);
- queryModel.setVectorReader(true);
- try {
- queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
- iterator = (AbstractDetailQueryResultIterator) queryExecutor.execute(queryModel);
- } catch (QueryExecutionException e) {
- throw new InterruptedException(e.getMessage());
- }
- }
-
- @Override public void close() throws IOException {
- logStatistics(rowCount, queryModel.getStatisticsRecorder());
- if (columnarBatch != null) {
- columnarBatch = null;
- }
- // clear dictionary cache
- Map<String, Dictionary> columnToDictionaryMapping = queryModel.getColumnToDictionaryMapping();
- if (null != columnToDictionaryMapping) {
- for (Map.Entry<String, Dictionary> entry : columnToDictionaryMapping.entrySet()) {
- CarbonUtil.clearDictionaryCache(entry.getValue());
- }
- }
- try {
- queryExecutor.finish();
- } catch (QueryExecutionException e) {
- throw new IOException(e);
- }
- }
-
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
- resultBatch();
-
- if (returnColumnarBatch) return nextBatch();
-
- if (batchIdx >= numBatched) {
- if (!nextBatch()) return false;
- }
- ++batchIdx;
- return true;
- }
-
- @Override public Object getCurrentValue() throws IOException, InterruptedException {
- if (returnColumnarBatch) {
- rowCount += columnarBatch.numValidRows();
- return columnarBatch;
- } else {
- return null;
- }
- }
-
- @Override public Void getCurrentKey() throws IOException, InterruptedException {
- return null;
- }
-
- @Override public float getProgress() throws IOException, InterruptedException {
- // TODO : Implement it based on total number of rows it is going to retrive.
- return 0;
- }
-
- /**
- * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
- * This object is reused. Calling this enables the vectorized reader. This should be called
- * before any calls to nextKeyValue/nextBatch.
- */
-
- private void initBatch() {
- List<QueryDimension> queryDimension = queryModel.getQueryDimension();
- List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
- StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
- for (int i = 0; i < queryDimension.size(); i++) {
- QueryDimension dim = queryDimension.get(i);
- if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(dim.getDimension().getDataType());
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- generator.getReturnType());
- } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- dim.getDimension().getDataType());
- } else if (dim.getDimension().isComplex()) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- dim.getDimension().getDataType());
- } else {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
- DataTypes.INT);
- }
- }
-
- for (QueryMeasure msr : queryMeasures) {
- DataType dataType = msr.getMeasure().getDataType();
- if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT
- || dataType == DataTypes.LONG) {
- fields[msr.getQueryOrder()] =
- new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
- } else if (DataTypes.isDecimal(dataType)) {
- fields[msr.getQueryOrder()] =
- new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
- } else {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
- }
- }
-
- columnarBatch = CarbonVectorBatch.allocate(fields);
- CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
- boolean[] filteredRows = new boolean[columnarBatch.capacity()];
- for (int i = 0; i < fields.length; i++) {
- vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
- }
- carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
- }
-
-
- private CarbonVectorBatch resultBatch() {
- if (columnarBatch == null) initBatch();
- return columnarBatch;
- }
-
- /*
- * Can be called before any rows are returned to enable returning columnar batches directly.
- */
- private void enableReturningBatches() {
- returnColumnarBatch = true;
- }
-
- /**
- * Advances to the next batch of rows. Returns false if there are no more.
- */
- private boolean nextBatch() {
- columnarBatch.reset();
- carbonColumnarBatch.reset();
- if (iterator.hasNext()) {
- iterator.processNextBatch(carbonColumnarBatch);
- int actualSize = carbonColumnarBatch.getActualSize();
- columnarBatch.setNumRows(actualSize);
- numBatched = actualSize;
- batchIdx = 0;
- return true;
- }
- return false;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
index 1679f29..5f1f90a 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSource.java
@@ -54,7 +54,7 @@ class CarbondataPageSource implements ConnectorPageSource {
private final List<Type> types;
private final PageBuilder pageBuilder;
private boolean closed;
- private CarbonVectorizedRecordReader vectorReader;
+ private PrestoCarbonVectorizedRecordReader vectorReader;
private CarbonDictionaryDecodeReadSupport<Object[]> readSupport;
private long sizeOfData = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
index c614fa9..5772fbf 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordCursor.java
@@ -54,7 +54,7 @@ public class CarbondataRecordCursor implements RecordCursor {
private CarbondataSplit split;
private CarbonDictionaryDecodeReadSupport readSupport;
private Tuple3<DataType, Dictionary, Int>[] dictionary;
- CarbonVectorizedRecordReader vectorizedRecordReader;
+ PrestoCarbonVectorizedRecordReader vectorizedRecordReader;
private long totalBytes;
private long nanoStart;
@@ -63,7 +63,7 @@ public class CarbondataRecordCursor implements RecordCursor {
public CarbondataRecordCursor(CarbonDictionaryDecodeReadSupport readSupport,
- CarbonVectorizedRecordReader vectorizedRecordReader,
+ PrestoCarbonVectorizedRecordReader vectorizedRecordReader,
List<CarbondataColumnHandle> columnHandles,
CarbondataSplit split) {
this.vectorizedRecordReader = vectorizedRecordReader;
@@ -194,7 +194,7 @@ public class CarbondataRecordCursor implements RecordCursor {
//todo delete cache from readSupport
}
- public CarbonVectorizedRecordReader getVectorizedRecordReader() {
+ public PrestoCarbonVectorizedRecordReader getVectorizedRecordReader() {
return vectorizedRecordReader;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
index 0f8fe87..286ff0e 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSet.java
@@ -78,8 +78,8 @@ public class CarbondataRecordSet implements RecordSet {
readSupport
.initialize(queryModel.getProjectionColumns(), queryModel.getTable());
CarbonIterator iterator = queryExecutor.execute(queryModel);
- CarbonVectorizedRecordReader vectorReader =
- new CarbonVectorizedRecordReader(queryExecutor, queryModel,
+ PrestoCarbonVectorizedRecordReader vectorReader =
+ new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
(AbstractDetailQueryResultIterator) iterator);
return new CarbondataRecordCursor(readSupport, vectorReader, columns, split);
} catch (QueryExecutionException e) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
index f039daf..5a2f831 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataRecordSetProvider.java
@@ -105,7 +105,7 @@ public class CarbondataRecordSetProvider implements ConnectorRecordSetProvider {
new TaskAttemptContextImpl(jobConf, new TaskAttemptID("", 1, TaskType.MAP, 0, 0));
CarbonInputSplit carbonInputSplit =
CarbonLocalInputSplit.convertSplit(carbondataSplit.getLocalInputSplit());
- queryModel = carbonTableInputFormat.getQueryModel(carbonInputSplit, hadoopAttemptContext);
+ queryModel = carbonTableInputFormat.createQueryModel(carbonInputSplit, hadoopAttemptContext);
queryModel.setVectorReader(true);
} catch (IOException e) {
throw new RuntimeException("Unable to get the Query Model ", e);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
new file mode 100644
index 0000000..a1907db
--- /dev/null
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoCarbonVectorizedRecordReader.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.presto;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.hadoop.AbstractRecordReader;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * A specialized RecordReader that reads into InternalRows or ColumnarBatches directly using the
+ * carbondata column APIs and fills the data directly into columns.
+ */
+class PrestoCarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
+
+ private int batchIdx = 0;
+
+ private int numBatched = 0;
+
+ private CarbonVectorBatch columnarBatch;
+
+ private CarbonColumnarBatch carbonColumnarBatch;
+
+ /**
+ * If true, this class returns batches instead of rows.
+ */
+ private boolean returnColumnarBatch;
+
+ private QueryModel queryModel;
+
+ private AbstractDetailQueryResultIterator iterator;
+
+ private QueryExecutor queryExecutor;
+
+ public PrestoCarbonVectorizedRecordReader(QueryExecutor queryExecutor, QueryModel queryModel, AbstractDetailQueryResultIterator iterator) {
+ this.queryModel = queryModel;
+ this.iterator = iterator;
+ this.queryExecutor = queryExecutor;
+ enableReturningBatches();
+ }
+
+ /**
+ * Implementation of RecordReader API.
+ */
+ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ throws IOException, InterruptedException, UnsupportedOperationException {
+ // The input split can contain single HDFS block or multiple blocks, so firstly get all the
+ // blocks and then set them in the query model.
+ List<CarbonInputSplit> splitList;
+ if (inputSplit instanceof CarbonInputSplit) {
+ splitList = new ArrayList<>(1);
+ splitList.add((CarbonInputSplit) inputSplit);
+ } else if (inputSplit instanceof CarbonMultiBlockSplit) {
+ // contains multiple blocks, this is an optimization for concurrent query.
+ CarbonMultiBlockSplit multiBlockSplit = (CarbonMultiBlockSplit) inputSplit;
+ splitList = multiBlockSplit.getAllSplits();
+ } else {
+ throw new RuntimeException("unsupported input split type: " + inputSplit);
+ }
+ List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
+ queryModel.setTableBlockInfos(tableBlockInfoList);
+ queryModel.setVectorReader(true);
+ try {
+ queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+ iterator = (AbstractDetailQueryResultIterator) queryExecutor.execute(queryModel);
+ } catch (QueryExecutionException e) {
+ throw new InterruptedException(e.getMessage());
+ }
+ }
+
+ @Override public void close() throws IOException {
+ logStatistics(rowCount, queryModel.getStatisticsRecorder());
+ if (columnarBatch != null) {
+ columnarBatch = null;
+ }
+ // clear dictionary cache
+ Map<String, Dictionary> columnToDictionaryMapping = queryModel.getColumnToDictionaryMapping();
+ if (null != columnToDictionaryMapping) {
+ for (Map.Entry<String, Dictionary> entry : columnToDictionaryMapping.entrySet()) {
+ CarbonUtil.clearDictionaryCache(entry.getValue());
+ }
+ }
+ try {
+ queryExecutor.finish();
+ } catch (QueryExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ resultBatch();
+
+ if (returnColumnarBatch) return nextBatch();
+
+ if (batchIdx >= numBatched) {
+ if (!nextBatch()) return false;
+ }
+ ++batchIdx;
+ return true;
+ }
+
+ @Override public Object getCurrentValue() throws IOException, InterruptedException {
+ if (returnColumnarBatch) {
+ rowCount += columnarBatch.numValidRows();
+ return columnarBatch;
+ } else {
+ return null;
+ }
+ }
+
+ @Override public Void getCurrentKey() throws IOException, InterruptedException {
+ return null;
+ }
+
+ @Override public float getProgress() throws IOException, InterruptedException {
+ // TODO : Implement it based on total number of rows it is going to retrive.
+ return 0;
+ }
+
+ /**
+ * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
+ * This object is reused. Calling this enables the vectorized reader. This should be called
+ * before any calls to nextKeyValue/nextBatch.
+ */
+
+ private void initBatch() {
+ List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
+ List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
+ StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
+ for (int i = 0; i < queryDimension.size(); i++) {
+ ProjectionDimension dim = queryDimension.get(i);
+ if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dim.getDimension().getDataType());
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ generator.getReturnType());
+ } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ dim.getDimension().getDataType());
+ } else if (dim.getDimension().isComplex()) {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ dim.getDimension().getDataType());
+ } else {
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
+ DataTypes.INT);
+ }
+ }
+
+ for (ProjectionMeasure msr : queryMeasures) {
+ DataType dataType = msr.getMeasure().getDataType();
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT
+ || dataType == DataTypes.LONG) {
+ fields[msr.getOrdinal()] =
+ new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
+ } else if (DataTypes.isDecimal(dataType)) {
+ fields[msr.getOrdinal()] =
+ new StructField(msr.getColumnName(), msr.getMeasure().getDataType());
+ } else {
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE);
+ }
+ }
+
+ columnarBatch = CarbonVectorBatch.allocate(fields);
+ CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
+ boolean[] filteredRows = new boolean[columnarBatch.capacity()];
+ for (int i = 0; i < fields.length; i++) {
+ vectors[i] = new CarbonColumnVectorWrapper(columnarBatch.column(i), filteredRows);
+ }
+ carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
+ }
+
+
+ private CarbonVectorBatch resultBatch() {
+ if (columnarBatch == null) initBatch();
+ return columnarBatch;
+ }
+
+ /*
+ * Can be called before any rows are returned to enable returning columnar batches directly.
+ */
+ private void enableReturningBatches() {
+ returnColumnarBatch = true;
+ }
+
+ /**
+ * Advances to the next batch of rows. Returns false if there are no more.
+ */
+ private boolean nextBatch() {
+ columnarBatch.reset();
+ carbonColumnarBatch.reset();
+ if (iterator.hasNext()) {
+ iterator.processNextBatch(carbonColumnarBatch);
+ int actualSize = carbonColumnarBatch.getActualSize();
+ columnarBatch.setNumRows(actualSize);
+ numBatched = actualSize;
+ batchIdx = 0;
+ return true;
+ }
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index ab3ab5d..3c70619 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -171,7 +171,8 @@ class CarbonMergerRDD[K, V](
LOGGER.info(s"Restructured block exists: $restructuredBlockExists")
DataTypeUtil.setDataTypeConverter(new SparkDataTypeConverterImpl)
exec = new CarbonCompactionExecutor(segmentMapping, segmentProperties,
- carbonTable, dataFileMetadataSegMapping, restructuredBlockExists)
+ carbonTable, dataFileMetadataSegMapping, restructuredBlockExists,
+ new SparkDataTypeConverterImpl)
// fire a query and get the results.
var result2: java.util.List[RawResultIterator] = null
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 772f702..97be1fb 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -336,7 +336,7 @@ class CarbonScanRDD(
TaskMetricsMap.getInstance().registerThreadCallback()
inputMetricsStats.initBytesReadCallback(context, inputSplit)
val iterator = if (inputSplit.getAllSplits.size() > 0) {
- val model = format.getQueryModel(inputSplit, attemptContext)
+ val model = format.createQueryModel(inputSplit, attemptContext)
// get RecordReader by FileFormat
val reader: RecordReader[Void, Object] = inputSplit.getFileFormat match {
case FileFormat.ROW_V1 =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index 7d42130..432d50a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -40,7 +40,7 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
private DataType blockDataType;
- public ColumnarVectorWrapper(ColumnVector columnVector, boolean[] filteredRows) {
+ ColumnarVectorWrapper(ColumnVector columnVector, boolean[] filteredRows) {
this.columnVector = columnVector;
this.filteredRows = filteredRows;
this.dataType = CarbonScalaUtil.convertSparkToCarbonDataType(columnVector.dataType());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 5d927df..73da878 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -35,8 +35,8 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.core.scan.model.QueryDimension;
-import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.ProjectionDimension;
+import org.apache.carbondata.core.scan.model.ProjectionMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
@@ -100,7 +100,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
/**
* Implementation of RecordReader API.
*/
- @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException, UnsupportedOperationException {
// The input split can contain single HDFS block or multiple blocks, so firstly get all the
// blocks and then set them in the query model.
@@ -145,7 +146,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
}
}
- @Override public void close() throws IOException {
+ @Override
+ public void close() throws IOException {
logStatistics(rowCount, queryModel.getStatisticsRecorder());
if (columnarBatch != null) {
columnarBatch.close();
@@ -165,10 +167,13 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
}
}
- @Override public boolean nextKeyValue() throws IOException, InterruptedException {
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
resultBatch();
- if (returnColumnarBatch) return nextBatch();
+ if (returnColumnarBatch) {
+ return nextBatch();
+ }
if (batchIdx >= numBatched) {
if (!nextBatch()) return false;
@@ -177,7 +182,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
return true;
}
- @Override public Object getCurrentValue() throws IOException, InterruptedException {
+ @Override
+ public Object getCurrentValue() throws IOException, InterruptedException {
if (returnColumnarBatch) {
int value = columnarBatch.numValidRows();
rowCount += value;
@@ -190,11 +196,13 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
return columnarBatch.getRow(batchIdx - 1);
}
- @Override public Void getCurrentKey() throws IOException, InterruptedException {
+ @Override
+ public Void getCurrentKey() throws IOException, InterruptedException {
return null;
}
- @Override public float getProgress() throws IOException, InterruptedException {
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
// TODO : Implement it based on total number of rows it is going to retrive.
return 0;
}
@@ -206,44 +214,44 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
*/
private void initBatch(MemoryMode memMode) {
- List<QueryDimension> queryDimension = queryModel.getQueryDimension();
- List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
+ List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
+ List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
for (int i = 0; i < queryDimension.size(); i++) {
- QueryDimension dim = queryDimension.get(i);
+ ProjectionDimension dim = queryDimension.get(i);
if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory
.getDirectDictionaryGenerator(dim.getDimension().getDataType());
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
} else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
null);
} else if (dim.getDimension().isComplex()) {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true,
null);
} else {
- fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(),
+ fields[dim.getOrdinal()] = new StructField(dim.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.INT), true, null);
}
}
for (int i = 0; i < queryMeasures.size(); i++) {
- QueryMeasure msr = queryMeasures.get(i);
+ ProjectionMeasure msr = queryMeasures.get(i);
DataType dataType = msr.getMeasure().getDataType();
if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT ||
dataType == DataTypes.INT || dataType == DataTypes.LONG) {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true,
null);
} else if (DataTypes.isDecimal(dataType)) {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true,
null);
} else {
- fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
+ fields[msr.getOrdinal()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.DOUBLE), true, null);
}
}
@@ -261,9 +269,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
initBatch(DEFAULT_MEMORY_MODE);
}
- private ColumnarBatch resultBatch() {
+ private void resultBatch() {
if (columnarBatch == null) initBatch();
- return columnarBatch;
}
/*
[27/54] [abbrv] carbondata git commit: [CARBONDATA-2159] Remove
carbon-spark dependency in store-sdk module
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 8d394db..e69de29 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -1,610 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.util
-
-import java.text.SimpleDateFormat
-import java.util
-import java.util.{Date, List, Locale}
-
-import scala.collection.{immutable, mutable}
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.commons.lang3.StringUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
-import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.hadoop.mapreduce.task.{JobContextImpl, TaskAttemptContextImpl}
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
-import org.apache.spark.sql.execution.datasources.{FilePartition, FileScanRDD, PartitionedFile}
-import org.apache.spark.sql.util.CarbonException
-import org.apache.spark.sql.util.SparkSQLUtil.sessionState
-
-import org.apache.carbondata.common.constants.LoggerAction
-import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
-import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
-import org.apache.carbondata.core.indexstore.PartitionSpec
-import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.processing.loading.constants.DataLoadProcessorConstants
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
-import org.apache.carbondata.processing.util.{CarbonLoaderUtil, DeleteLoadFolders, TableOptionConstant}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark.LOGGER
-import org.apache.carbondata.spark.load.ValidateUtil
-import org.apache.carbondata.spark.rdd.SerializableConfiguration
-
-/**
- * the util object of data loading
- */
-object DataLoadingUtil {
-
- val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
-
- /**
- * get data loading options and initialise default value
- */
- def getDataLoadingOptions(
- carbonProperty: CarbonProperties,
- options: immutable.Map[String, String]): mutable.Map[String, String] = {
- val optionsFinal = scala.collection.mutable.Map[String, String]()
- optionsFinal.put("delimiter", options.getOrElse("delimiter", ","))
- optionsFinal.put("quotechar", options.getOrElse("quotechar", "\""))
- optionsFinal.put("fileheader", options.getOrElse("fileheader", ""))
- optionsFinal.put("commentchar", options.getOrElse("commentchar", "#"))
- optionsFinal.put("columndict", options.getOrElse("columndict", null))
-
- optionsFinal.put("escapechar",
- CarbonLoaderUtil.getEscapeChar(options.getOrElse("escapechar", "\\")))
-
- optionsFinal.put(
- "serialization_null_format",
- options.getOrElse("serialization_null_format", "\\N"))
-
- optionsFinal.put(
- "bad_records_logger_enable",
- options.getOrElse(
- "bad_records_logger_enable",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE,
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)))
-
- val badRecordActionValue = carbonProperty.getProperty(
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
- CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)
-
- optionsFinal.put(
- "bad_records_action",
- options.getOrElse(
- "bad_records_action",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION,
- badRecordActionValue)))
-
- optionsFinal.put(
- "is_empty_data_bad_record",
- options.getOrElse(
- "is_empty_data_bad_record",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD,
- CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)))
-
- optionsFinal.put(
- "skip_empty_line",
- options.getOrElse(
- "skip_empty_line",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)))
-
- optionsFinal.put("all_dictionary_path", options.getOrElse("all_dictionary_path", ""))
-
- optionsFinal.put(
- "complex_delimiter_level_1",
- options.getOrElse("complex_delimiter_level_1", "$"))
-
- optionsFinal.put(
- "complex_delimiter_level_2",
- options.getOrElse("complex_delimiter_level_2", ":"))
-
- optionsFinal.put(
- "dateformat",
- options.getOrElse(
- "dateformat",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)))
-
- optionsFinal.put(
- "timestampformat",
- options.getOrElse(
- "timestampformat",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT,
- CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)))
-
- optionsFinal.put(
- "global_sort_partitions",
- options.getOrElse(
- "global_sort_partitions",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS,
- null)))
-
- optionsFinal.put("maxcolumns", options.getOrElse("maxcolumns", null))
-
- optionsFinal.put(
- "batch_sort_size_inmb",
- options.getOrElse(
- "batch_sort_size_inmb",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BATCH_SORT_SIZE_INMB,
- carbonProperty.getProperty(
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB,
- CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB_DEFAULT))))
-
- optionsFinal.put(
- "bad_record_path",
- options.getOrElse(
- "bad_record_path",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH,
- carbonProperty.getProperty(
- CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))))
-
- val useOnePass = options.getOrElse(
- "single_pass",
- carbonProperty.getProperty(
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS,
- CarbonLoadOptionConstants.CARBON_OPTIONS_SINGLE_PASS_DEFAULT)).trim.toLowerCase match {
- case "true" =>
- true
- case "false" =>
- // when single_pass = false and if either alldictionarypath
- // or columnDict is configured the do not allow load
- if (StringUtils.isNotEmpty(optionsFinal("all_dictionary_path")) ||
- StringUtils.isNotEmpty(optionsFinal("columndict"))) {
- throw new MalformedCarbonCommandException(
- "Can not use all_dictionary_path or columndict without single_pass.")
- } else {
- false
- }
- case illegal =>
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- LOGGER.error(s"Can't use single_pass, because illegal syntax found: [$illegal] " +
- "Please set it as 'true' or 'false'")
- false
- }
- optionsFinal.put("single_pass", useOnePass.toString)
- optionsFinal
- }
-
- /**
- * check whether using default value or not
- */
- private def checkDefaultValue(value: String, default: String) = {
- if (StringUtils.isEmpty(value)) {
- default
- } else {
- value
- }
- }
-
- /**
- * build CarbonLoadModel for data loading
- * @param table CarbonTable object containing all metadata information for the table
- * like table name, table path, schema, etc
- * @param options Load options from user input
- * @return a new CarbonLoadModel instance
- */
- def buildCarbonLoadModelJava(
- table: CarbonTable,
- options: java.util.Map[String, String]
- ): CarbonLoadModel = {
- val carbonProperty: CarbonProperties = CarbonProperties.getInstance
- val optionsFinal = getDataLoadingOptions(carbonProperty, options.asScala.toMap)
- optionsFinal.put("sort_scope", "no_sort")
- if (!options.containsKey("fileheader")) {
- val csvHeader = table.getCreateOrderColumn(table.getTableName)
- .asScala.map(_.getColName).mkString(",")
- optionsFinal.put("fileheader", csvHeader)
- }
- val model = new CarbonLoadModel()
- buildCarbonLoadModel(
- table = table,
- carbonProperty = carbonProperty,
- options = options.asScala.toMap,
- optionsFinal = optionsFinal,
- carbonLoadModel = model,
- hadoopConf = null) // we have provided 'fileheader', so it can be null
-
- // set default values
- model.setTimestampformat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
- model.setDateFormat(CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT)
- model.setUseOnePass(options.asScala.getOrElse("onepass", "false").toBoolean)
- model.setDictionaryServerHost(options.asScala.getOrElse("dicthost", null))
- model.setDictionaryServerPort(options.asScala.getOrElse("dictport", "-1").toInt)
- model
- }
-
- /**
- * build CarbonLoadModel for data loading
- * @param table CarbonTable object containing all metadata information for the table
- * like table name, table path, schema, etc
- * @param carbonProperty Carbon property instance
- * @param options Load options from user input
- * @param optionsFinal Load options that populated with default values for optional options
- * @param carbonLoadModel The output load model
- * @param hadoopConf hadoopConf is needed to read CSV header if there 'fileheader' is not set in
- * user provided load options
- */
- def buildCarbonLoadModel(
- table: CarbonTable,
- carbonProperty: CarbonProperties,
- options: immutable.Map[String, String],
- optionsFinal: mutable.Map[String, String],
- carbonLoadModel: CarbonLoadModel,
- hadoopConf: Configuration,
- partition: Map[String, Option[String]] = Map.empty,
- isDataFrame: Boolean = false): Unit = {
- carbonLoadModel.setTableName(table.getTableName)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTablePath(table.getTablePath)
- carbonLoadModel.setTableName(table.getTableName)
- val dataLoadSchema = new CarbonDataLoadSchema(table)
- // Need to fill dimension relation
- carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
- val sort_scope = optionsFinal("sort_scope")
- val single_pass = optionsFinal("single_pass")
- val bad_records_logger_enable = optionsFinal("bad_records_logger_enable")
- val bad_records_action = optionsFinal("bad_records_action")
- var bad_record_path = optionsFinal("bad_record_path")
- val global_sort_partitions = optionsFinal("global_sort_partitions")
- val timestampformat = optionsFinal("timestampformat")
- val dateFormat = optionsFinal("dateformat")
- val delimeter = optionsFinal("delimiter")
- val complex_delimeter_level1 = optionsFinal("complex_delimiter_level_1")
- val complex_delimeter_level2 = optionsFinal("complex_delimiter_level_2")
- val all_dictionary_path = optionsFinal("all_dictionary_path")
- val column_dict = optionsFinal("columndict")
- ValidateUtil.validateDateTimeFormat(timestampformat, "TimestampFormat")
- ValidateUtil.validateDateTimeFormat(dateFormat, "DateFormat")
- ValidateUtil.validateSortScope(table, sort_scope)
-
- if (bad_records_logger_enable.toBoolean ||
- LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
- if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
- CarbonException.analysisException("Invalid bad records location.")
- }
- bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path)
- }
- carbonLoadModel.setBadRecordsLocation(bad_record_path)
-
- ValidateUtil.validateGlobalSortPartitions(global_sort_partitions)
- carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal("escapechar"), "\\"))
- carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal("quotechar"), "\""))
- carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal("commentchar"), "#"))
-
- // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
- // we should use table schema to generate file header.
- var fileHeader = optionsFinal("fileheader")
- val headerOption = options.get("header")
- if (headerOption.isDefined) {
- // whether the csv file has file header
- // the default value is true
- val header = try {
- headerOption.get.toBoolean
- } catch {
- case ex: IllegalArgumentException =>
- throw new MalformedCarbonCommandException(
- "'header' option should be either 'true' or 'false'. " + ex.getMessage)
- }
- if (header) {
- if (fileHeader.nonEmpty) {
- throw new MalformedCarbonCommandException(
- "When 'header' option is true, 'fileheader' option is not required.")
- }
- } else {
- if (fileHeader.isEmpty) {
- fileHeader = table.getCreateOrderColumn(table.getTableName)
- .asScala.map(_.getColName).mkString(",")
- }
- }
- }
-
- carbonLoadModel.setTimestampformat(timestampformat)
- carbonLoadModel.setDateFormat(dateFormat)
- carbonLoadModel.setDefaultTimestampFormat(carbonProperty.getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
-
- carbonLoadModel.setDefaultDateFormat(carbonProperty.getProperty(
- CarbonCommonConstants.CARBON_DATE_FORMAT,
- CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
-
- carbonLoadModel.setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," +
- optionsFinal("serialization_null_format"))
-
- carbonLoadModel.setBadRecordsLoggerEnable(
- TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + bad_records_logger_enable)
-
- carbonLoadModel.setBadRecordsAction(
- TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action.toUpperCase)
-
- carbonLoadModel.setIsEmptyDataBadRecord(
- DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
- optionsFinal("is_empty_data_bad_record"))
-
- carbonLoadModel.setSkipEmptyLine(optionsFinal("skip_empty_line"))
-
- carbonLoadModel.setSortScope(sort_scope)
- carbonLoadModel.setBatchSortSizeInMb(optionsFinal("batch_sort_size_inmb"))
- carbonLoadModel.setGlobalSortPartitions(global_sort_partitions)
- carbonLoadModel.setUseOnePass(single_pass.toBoolean)
-
- if (delimeter.equalsIgnoreCase(complex_delimeter_level1) ||
- complex_delimeter_level1.equalsIgnoreCase(complex_delimeter_level2) ||
- delimeter.equalsIgnoreCase(complex_delimeter_level2)) {
- CarbonException.analysisException(s"Field Delimiter and Complex types delimiter are same")
- } else {
- carbonLoadModel.setComplexDelimiterLevel1(complex_delimeter_level1)
- carbonLoadModel.setComplexDelimiterLevel2(complex_delimeter_level2)
- }
- // set local dictionary path, and dictionary file extension
- carbonLoadModel.setAllDictPath(all_dictionary_path)
- carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter))
- carbonLoadModel.setCsvHeader(fileHeader)
- carbonLoadModel.setColDictFilePath(column_dict)
-
- val ignoreColumns = new util.ArrayList[String]()
- if (!isDataFrame) {
- ignoreColumns.addAll(partition.filter(_._2.isDefined).keys.toList.asJava)
- }
- carbonLoadModel.setCsvHeaderColumns(
- CommonUtil.getCsvHeaderColumns(carbonLoadModel, hadoopConf, ignoreColumns))
-
- val validatedMaxColumns = CommonUtil.validateMaxColumns(
- carbonLoadModel.getCsvHeaderColumns,
- optionsFinal("maxcolumns"))
-
- carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
- if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
- }
- }
-
- private def isLoadDeletionRequired(metaDataLocation: String): Boolean = {
- val details = SegmentStatusManager.readLoadMetadata(metaDataLocation)
- if (details != null && details.nonEmpty) for (oneRow <- details) {
- if ((SegmentStatus.MARKED_FOR_DELETE == oneRow.getSegmentStatus ||
- SegmentStatus.COMPACTED == oneRow.getSegmentStatus ||
- SegmentStatus.INSERT_IN_PROGRESS == oneRow.getSegmentStatus ||
- SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS == oneRow.getSegmentStatus) &&
- oneRow.getVisibility.equalsIgnoreCase("true")) {
- return true
- }
- }
- false
- }
-
- def deleteLoadsAndUpdateMetadata(
- isForceDeletion: Boolean,
- carbonTable: CarbonTable,
- specs: util.List[PartitionSpec]): Unit = {
- if (isLoadDeletionRequired(carbonTable.getMetadataPath)) {
- val absoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier
-
- val (details, updationRequired) =
- isUpdationRequired(
- isForceDeletion,
- carbonTable,
- absoluteTableIdentifier)
-
-
- if (updationRequired) {
- val carbonTableStatusLock =
- CarbonLockFactory.getCarbonLockObj(
- absoluteTableIdentifier,
- LockUsage.TABLE_STATUS_LOCK
- )
- var locked = false
- var updationCompletionStaus = false
- try {
- // Update load metadate file after cleaning deleted nodes
- locked = carbonTableStatusLock.lockWithRetries()
- if (locked) {
- LOGGER.info("Table status lock has been successfully acquired.")
- // Again read status and check to verify updation required or not.
- val (details, updationRequired) =
- isUpdationRequired(
- isForceDeletion,
- carbonTable,
- absoluteTableIdentifier)
- if (!updationRequired) {
- return
- }
- // read latest table status again.
- val latestMetadata = SegmentStatusManager
- .readLoadMetadata(carbonTable.getMetadataPath)
-
- // update the metadata details from old to new status.
- val latestStatus = CarbonLoaderUtil
- .updateLoadMetadataFromOldToNew(details, latestMetadata)
-
- CarbonLoaderUtil.writeLoadMetadata(absoluteTableIdentifier, latestStatus)
- } else {
- val dbName = absoluteTableIdentifier.getCarbonTableIdentifier.getDatabaseName
- val tableName = absoluteTableIdentifier.getCarbonTableIdentifier.getTableName
- val errorMsg = "Clean files request is failed for " +
- s"$dbName.$tableName" +
- ". Not able to acquire the table status lock due to other operation " +
- "running in the background."
- LOGGER.audit(errorMsg)
- LOGGER.error(errorMsg)
- throw new Exception(errorMsg + " Please try after some time.")
- }
- updationCompletionStaus = true
- } finally {
- if (locked) {
- CarbonLockUtil.fileUnlock(carbonTableStatusLock, LockUsage.TABLE_STATUS_LOCK)
- }
- }
- if (updationCompletionStaus) {
- DeleteLoadFolders
- .physicalFactAndMeasureMetadataDeletion(absoluteTableIdentifier,
- carbonTable.getMetadataPath, isForceDeletion, specs)
- }
- }
- }
- }
-
- private def isUpdationRequired(isForceDeletion: Boolean,
- carbonTable: CarbonTable,
- absoluteTableIdentifier: AbsoluteTableIdentifier): (Array[LoadMetadataDetails], Boolean) = {
- val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath)
- // Delete marked loads
- val isUpdationRequired =
- DeleteLoadFolders.deleteLoadFoldersFromFileSystem(
- absoluteTableIdentifier,
- isForceDeletion,
- details,
- carbonTable.getMetadataPath
- )
- (details, isUpdationRequired)
- }
-
- /**
- * creates a RDD that does reading of multiple CSV files
- */
- def csvFileScanRDD(
- spark: SparkSession,
- model: CarbonLoadModel,
- hadoopConf: Configuration
- ): RDD[InternalRow] = {
- // 1. partition
- val defaultMaxSplitBytes = sessionState(spark).conf.filesMaxPartitionBytes
- val openCostInBytes = sessionState(spark).conf.filesOpenCostInBytes
- val defaultParallelism = spark.sparkContext.defaultParallelism
- CommonUtil.configureCSVInputFormat(hadoopConf, model)
- hadoopConf.set(FileInputFormat.INPUT_DIR, model.getFactFilePath)
- val jobConf = new JobConf(hadoopConf)
- SparkHadoopUtil.get.addCredentials(jobConf)
- val jobContext = new JobContextImpl(jobConf, null)
- val inputFormat = new CSVInputFormat()
- val rawSplits = inputFormat.getSplits(jobContext).toArray
- val splitFiles = rawSplits.map { split =>
- val fileSplit = split.asInstanceOf[FileSplit]
- PartitionedFile(
- InternalRow.empty,
- fileSplit.getPath.toString,
- fileSplit.getStart,
- fileSplit.getLength,
- fileSplit.getLocations)
- }.sortBy(_.length)(implicitly[Ordering[Long]].reverse)
- val totalBytes = splitFiles.map(_.length + openCostInBytes).sum
- val bytesPerCore = totalBytes / defaultParallelism
-
- val maxSplitBytes = Math.min(defaultMaxSplitBytes, Math.max(openCostInBytes, bytesPerCore))
- LOGGER.info(s"Planning scan with bin packing, max size: $maxSplitBytes bytes, " +
- s"open cost is considered as scanning $openCostInBytes bytes.")
-
- val partitions = new ArrayBuffer[FilePartition]
- val currentFiles = new ArrayBuffer[PartitionedFile]
- var currentSize = 0L
-
- def closePartition(): Unit = {
- if (currentFiles.nonEmpty) {
- val newPartition =
- FilePartition(
- partitions.size,
- currentFiles.toArray.toSeq)
- partitions += newPartition
- }
- currentFiles.clear()
- currentSize = 0
- }
-
- splitFiles.foreach { file =>
- if (currentSize + file.length > maxSplitBytes) {
- closePartition()
- }
- // Add the given file to the current partition.
- currentSize += file.length + openCostInBytes
- currentFiles += file
- }
- closePartition()
-
- // 2. read function
- val serializableConfiguration = new SerializableConfiguration(jobConf)
- val readFunction = new (PartitionedFile => Iterator[InternalRow]) with Serializable {
- override def apply(file: PartitionedFile): Iterator[InternalRow] = {
- new Iterator[InternalRow] {
- val hadoopConf = serializableConfiguration.value
- val jobTrackerId: String = {
- val formatter = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
- formatter.format(new Date())
- }
- val attemptId = new TaskAttemptID(jobTrackerId, 0, TaskType.MAP, 0, 0)
- val hadoopAttemptContext = new TaskAttemptContextImpl(hadoopConf, attemptId)
- val inputSplit =
- new FileSplit(new Path(file.filePath), file.start, file.length, file.locations)
- var finished = false
- val inputFormat = new CSVInputFormat()
- val reader = inputFormat.createRecordReader(inputSplit, hadoopAttemptContext)
- reader.initialize(inputSplit, hadoopAttemptContext)
-
- override def hasNext: Boolean = {
- if (!finished) {
- if (reader != null) {
- if (reader.nextKeyValue()) {
- true
- } else {
- finished = true
- reader.close()
- false
- }
- } else {
- finished = true
- false
- }
- } else {
- false
- }
- }
-
- override def next(): InternalRow = {
- new GenericInternalRow(reader.getCurrentValue.get().asInstanceOf[Array[Any]])
- }
- }
- }
- }
- new FileScanRDD(spark, readFunction, partitions)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 2bd4f45..fb9ecac 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -321,7 +321,7 @@ object GlobalDictionaryUtil {
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
// get load count
if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
+ carbonLoadModel.readAndSetLoadMetadataDetails()
}
val absoluteTableIdentifier = AbsoluteTableIdentifier.from(carbonLoadModel.getTablePath, table)
DictionaryLoadModel(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 84215fd..b1e4083 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.PartitionUtils
import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataTypes
@@ -42,7 +43,6 @@ import org.apache.carbondata.core.metadata.schema.partition.PartitionType
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.util.CarbonLoaderUtil
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.{CommonUtil, DataTypeConverterUtil}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 349c436..2c4d604 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -153,7 +153,7 @@ object CarbonDataRDDFactory {
if (compactionModel.compactionType != CompactionType.IUD_UPDDEL_DELTA) {
// update the updated table status. For the case of Update Delta Compaction the Metadata
// is filled in LoadModel, no need to refresh.
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
+ carbonLoadModel.readAndSetLoadMetadataDetails()
}
val compactionThread = new Thread {
@@ -274,7 +274,7 @@ object CarbonDataRDDFactory {
loadModel.setTableName(table.getCarbonTableIdentifier.getTableName)
loadModel.setDatabaseName(table.getCarbonTableIdentifier.getDatabaseName)
loadModel.setTablePath(table.getTablePath)
- CommonUtil.readLoadMetadataDetails(loadModel)
+ loadModel.readAndSetLoadMetadataDetails()
val loadStartTime = CarbonUpdateUtil.readCurrentTime()
loadModel.setFactTimeStamp(loadStartTime)
loadModel
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 1210b92..231b748 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -76,7 +76,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
}
// scan again and determine if anything is there to merge again.
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
+ carbonLoadModel.readAndSetLoadMetadataDetails()
segList = carbonLoadModel.getLoadMetadataDetails
// in case of major compaction we will scan only once and come out as it will keep
// on doing major for the new loads also.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index 7d70534..adf5e04 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -35,13 +35,13 @@ import org.apache.spark.sql.streaming.OutputMode
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CarbonException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry
import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.spark.CarbonOption
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CarbonScalaUtil
import org.apache.carbondata.streaming.{CarbonStreamException, StreamSinkFactory}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 0eb7ad5..6508777 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -23,10 +23,10 @@ import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.preaaggregate.CreatePreAggregateTableCommand
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Below command class will be used to create datamap on table
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 2675036..dcc71a2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, NoSuchDataMapException}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
@@ -34,7 +35,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.events._
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, NoSuchDataMapException}
/**
* Drops the datamap and any related tables associated with the datamap
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
index acc3358..e47c500 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAlterTableCompactionCommand.scala
@@ -21,18 +21,20 @@ import java.io.{File, IOException}
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{CarbonEnv, Row, SparkSession, SQLContext}
+import org.apache.spark.sql.{CarbonEnv, Row, SQLContext, SparkSession}
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.command.{AlterTableModel, AtomicRunnableCommand, CarbonMergerMapping, CompactionModel, DataCommand}
+import org.apache.spark.sql.execution.command.{AlterTableModel, AtomicRunnableCommand, CarbonMergerMapping, CompactionModel}
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog}
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.AlterTableUtil
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
@@ -44,7 +46,6 @@ import org.apache.carbondata.events._
import org.apache.carbondata.processing.loading.events.LoadEvents.LoadMetadataEvent
import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CompactionType}
-import org.apache.carbondata.spark.exception.{ConcurrentOperationException, MalformedCarbonCommandException}
import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
import org.apache.carbondata.spark.util.CommonUtil
import org.apache.carbondata.streaming.StreamHandoffRDD
@@ -103,7 +104,6 @@ case class CarbonAlterTableCompactionCommand(
.fireEvent(alterTableCompactionExceptionEvent, operationContext)
compactionException = operationContext.getProperty("compactionException").toString
}
-
if (compactionException.equalsIgnoreCase("true") && null == compactionType) {
throw new MalformedCarbonCommandException(
"Unsupported alter operation on carbon table")
@@ -159,8 +159,7 @@ case class CarbonAlterTableCompactionCommand(
operationContext: OperationContext): Unit = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getName)
val compactionType = CompactionType.valueOf(alterTableModel.compactionType.toUpperCase)
- val compactionSize: Long = CarbonDataMergerUtil
- .getCompactionSize(compactionType, carbonLoadModel)
+ val compactionSize = CarbonDataMergerUtil.getCompactionSize(compactionType, carbonLoadModel)
if (CompactionType.IUD_UPDDEL_DELTA == compactionType) {
if (alterTableModel.segmentUpdateStatusManager.isDefined) {
carbonLoadModel.setSegmentUpdateStatusManager(
@@ -175,7 +174,7 @@ case class CarbonAlterTableCompactionCommand(
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
if (null == carbonLoadModel.getLoadMetadataDetails) {
- CommonUtil.readLoadMetadataDetails(carbonLoadModel)
+ carbonLoadModel.readAndSetLoadMetadataDetails()
}
if (compactionType == CompactionType.STREAMING) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index a42031d..70134a6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -46,6 +46,7 @@ import org.apache.spark.storage.StorageLevel
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -70,12 +71,15 @@ import org.apache.carbondata.processing.exception.DataLoadingException
import org.apache.carbondata.processing.loading.TableProcessingOperations
import org.apache.carbondata.processing.loading.events.LoadEvents.{LoadMetadataEvent, LoadTablePostExecutionEvent, LoadTablePreExecutionEvent}
import org.apache.carbondata.processing.loading.exception.NoRetryException
+import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
+import org.apache.carbondata.processing.util.CarbonLoaderUtil
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.sort.SortScopeOptions
import org.apache.carbondata.processing.util.{CarbonDataProcessorUtil, CarbonLoaderUtil}
import org.apache.carbondata.spark.dictionary.provider.SecureDictionaryServiceProvider
import org.apache.carbondata.spark.dictionary.server.SecureDictionaryServer
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, CarbonDropPartitionRDD}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, GlobalDictionaryUtil}
import org.apache.carbondata.spark.load.DataLoadProcessorStepOnSpark
import org.apache.carbondata.spark.rdd.CarbonDataRDDFactory
import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, GlobalDictionaryUtil, SparkDataTypeConverterImpl}
@@ -174,7 +178,7 @@ case class CarbonLoadDataCommand(
val carbonLoadModel = new CarbonLoadModel()
try {
val tableProperties = table.getTableInfo.getFactTable.getTableProperties
- val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
+ val optionsFinal = LoadOption.fillOptionWithDefaultValue(options.asJava)
optionsFinal.put("sort_scope", tableProperties.asScala.getOrElse("sort_scope",
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
carbonProperty.getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
@@ -189,10 +193,8 @@ case class CarbonLoadDataCommand(
carbonLoadModel.setAggLoadRequest(
internalOptions.getOrElse(CarbonCommonConstants.IS_INTERNAL_LOAD_CALL, "false").toBoolean)
carbonLoadModel.setSegmentId(internalOptions.getOrElse("mergedSegmentName", ""))
- DataLoadingUtil.buildCarbonLoadModel(
- table,
- carbonProperty,
- options,
+ new CarbonLoadModelBuilder(table).build(
+ options.asJava,
optionsFinal,
carbonLoadModel,
hadoopConf,
@@ -221,7 +223,7 @@ case class CarbonLoadDataCommand(
carbonLoadModel,
factPath,
dataFrame.isDefined,
- optionsFinal.asJava,
+ optionsFinal,
options.asJava,
isOverwriteTable)
operationContext.setProperty("isOverwrite", isOverwriteTable)
@@ -229,6 +231,7 @@ case class CarbonLoadDataCommand(
// First system has to partition the data first and then call the load data
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
// Clean up the old invalid segment data before creating a new entry for new load.
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(table, false)
DataLoadingUtil.deleteLoadsAndUpdateMetadata(
isForceDeletion = false,
table,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
index a37d6dc..f074285 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForDeleteCommand.scala
@@ -22,13 +22,13 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.carbondata.common.exceptions.ConcurrentOperationException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.mutate.CarbonUpdateUtil
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.events.{DeleteFromTablePostEvent, DeleteFromTablePreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.processing.loading.FailureCauses
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
/**
* IUD update delete and compaction framework.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
index 4886676..5165342 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.types.ArrayType
import org.apache.spark.storage.StorageLevel
+import org.apache.carbondata.common.exceptions.ConcurrentOperationException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.Segment
@@ -34,7 +35,6 @@ import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.{OperationContext, OperationListenerBus, UpdateTablePostEvent, UpdateTablePreEvent}
import org.apache.carbondata.processing.loading.FailureCauses
-import org.apache.carbondata.spark.exception.ConcurrentOperationException
private[sql] case class CarbonProjectForUpdateCommand(
plan: LogicalPlan,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
index 5817d88..220d75d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/IUDCommonUtil.scala
@@ -23,9 +23,9 @@ import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan}
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.HiveSessionCatalog
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
* Util for IUD common function
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index 4d0a4c5..c836584 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager}
-import org.apache.carbondata.spark.util.DataLoadingUtil
/**
* Below command class will be used to create pre-aggregate table
@@ -179,11 +178,7 @@ case class CreatePreAggregateTableCommand(
// This will be used to check if the parent table has any segments or not. If not then no
// need to fire load for pre-aggregate table. Therefore reading the load details for PARENT
// table.
- DataLoadingUtil.deleteLoadsAndUpdateMetadata(isForceDeletion = false,
- parentTable,
- CarbonFilters.getCurrentPartitions(sparkSession,
- TableIdentifier(parentTable.getTableName,
- Some(parentTable.getDatabaseName))).map(_.asJava).orNull)
+ SegmentStatusManager.deleteLoadsAndUpdateMetadata(parentTable, false)
val loadAvailable = SegmentStatusManager.readLoadMetadata(parentTable.getMetadataPath)
if (loadAvailable.exists(load => load.getSegmentStatus == SegmentStatus.INSERT_IN_PROGRESS ||
load.getSegmentStatus == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index ebf87d2..a5d256c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.spark.sql.types.DataType
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.locks.{CarbonLockUtil, ICarbonLock, LockUsage}
@@ -42,7 +43,6 @@ import org.apache.carbondata.core.metadata.schema.table.{AggregationDataMapSchem
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.format.TableInfo
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
index 9fde675..cf77e0f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableRenameCommand.scala
@@ -25,6 +25,8 @@ import org.apache.spark.sql.execution.command.{AlterTableRenameModel, MetadataCo
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog}
import org.apache.spark.util.AlterTableUtil
+import org.apache.carbondata.common.exceptions.ConcurrentOperationException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -37,7 +39,6 @@ import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.events.{AlterTableRenamePostEvent, AlterTableRenamePreEvent, OperationContext, OperationListenerBus}
import org.apache.carbondata.format.SchemaEvolutionEntry
-import org.apache.carbondata.spark.exception.{ConcurrentOperationException, MalformedCarbonCommandException}
private[sql] case class CarbonAlterTableRenameCommand(
alterTableRenameModel: AlterTableRenameModel)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
index 45767da..2b35416 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/timeseries/TimeSeriesUtil.scala
@@ -18,12 +18,12 @@ package org.apache.spark.sql.execution.command.timeseries
import org.apache.spark.sql.execution.command.{DataMapField, Field}
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
import org.apache.carbondata.core.metadata.schema.datamap.Granularity
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.preagg.TimeSeriesUDF
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, MalformedDataMapCommandException}
/**
* Utility class for time series to keep
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
index b4d3bea..61a31a5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/CarbonFileFormat.scala
@@ -50,6 +50,9 @@ import org.apache.carbondata.hadoop.api.{CarbonOutputCommitter, CarbonTableOutpu
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat.CarbonRecordWriter
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil
+import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable
+import org.apache.carbondata.processing.loading.model.{CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
+import org.apache.carbondata.spark.util.{CarbonScalaUtil, Util}
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.spark.util.{CarbonScalaUtil, DataLoadingUtil, Util}
@@ -87,7 +90,7 @@ with Serializable {
TableIdentifier(options("tableName"), options.get("dbName")))(sparkSession)
val model = new CarbonLoadModel
val carbonProperty = CarbonProperties.getInstance()
- val optionsFinal = DataLoadingUtil.getDataLoadingOptions(carbonProperty, options)
+ val optionsFinal = LoadOption.fillOptionWithDefaultValue(options.asJava)
val tableProperties = table.getTableInfo.getFactTable.getTableProperties
optionsFinal.put("sort_scope", tableProperties.asScala.getOrElse("sort_scope",
carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
@@ -102,14 +105,11 @@ with Serializable {
val optionsLocal = new mutable.HashMap[String, String]()
optionsLocal ++= options
optionsLocal += (("header", "false"))
- DataLoadingUtil.buildCarbonLoadModel(
- table,
- carbonProperty,
- optionsLocal.toMap,
+ new CarbonLoadModelBuilder(table).build(
+ optionsLocal.toMap.asJava,
optionsFinal,
model,
- conf
- )
+ conf)
model.setUseOnePass(options.getOrElse("onepass", "false").toBoolean)
model.setDictionaryServerHost(options.getOrElse("dicthost", null))
model.setDictionaryServerPort(options.getOrElse("dictport", "-1").toInt)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
index dcbce84..ec20ec2 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/DDLStrategy.scala
@@ -32,10 +32,10 @@ import org.apache.spark.sql.execution.datasources.RefreshTable
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.util.{CarbonReflectionUtils, FileUtils}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.merger.CompactionType
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
* Carbon strategies for ddl commands
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
index 608ec60..7028dcf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/StreamingTableStrategy.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.command.AlterTableRenameCommand
import org.apache.spark.sql.execution.command.mutation.{CarbonProjectForDeleteCommand, CarbonProjectForUpdateCommand}
import org.apache.spark.sql.execution.command.schema.{CarbonAlterTableAddColumnCommand, CarbonAlterTableDataTypeChangeCommand, CarbonAlterTableDropColumnCommand}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
/**
* Strategy for streaming table, like blocking unsupported operation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
index 7ca34af..27c7d17 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/execution/command/CarbonHiveCommands.scala
@@ -24,9 +24,9 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, SessionParams}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
case class CarbonDropDatabaseCommand(command: DropDatabaseCommand)
extends RunnableCommand {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
index 3896ce9..2905b60 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
@@ -36,9 +36,9 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.spark.CarbonOption
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
index ad6d0c7..ef4836e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala
@@ -32,10 +32,10 @@ import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.CarbonException
import org.apache.spark.util.CarbonReflectionUtils
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.hadoop.util.SchemaReader
import org.apache.carbondata.spark.CarbonOption
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index bc36e9c..aaa87a3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalog, HiveExternalCatalog}
import org.apache.spark.sql.hive.HiveExternalCatalog._
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
@@ -39,7 +40,6 @@ import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.path.CarbonTablePath.getNewTablePath
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
object AlterTableUtil {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
index bc62902..a8094b6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
@@ -19,10 +19,10 @@ package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
* table api util
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
index e82b485..f033a8e 100644
--- a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
+++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.hive
-
-import scala.collection.generic.SeqFactory
-
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
@@ -46,10 +43,8 @@ import org.apache.spark.sql.types.DecimalType
import org.apache.spark.util.CarbonReflectionUtils
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.DataMapStoreManager
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CarbonScalaUtil
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
index c676b01..5ade510 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/segmentreading/TestSegmentReading.scala
@@ -4,7 +4,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
/**
* Created by rahul on 19/9/17.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 1d41ddc..3298009 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.processing.util.TableOptionConstant
/**
@@ -63,7 +63,7 @@ class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
carbonLoadModel.setCsvHeaderColumns(
- CommonUtil.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
+ LoadOption.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
// Create table and metadata folders if not exist
val metadataDirectoryPath = CarbonTablePath.getMetadataPath(table.getTablePath)
val fileType = FileFactory.getFileType(metadataDirectoryPath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 7ca0b56..43d8c03 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -23,14 +23,14 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.{CarbonEnv, SparkSession}
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.processing.exception.DataLoadingException
-import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel}
+import org.apache.carbondata.processing.loading.model.{CarbonDataLoadSchema, CarbonLoadModel, CarbonLoadModelBuilder, LoadOption}
import org.apache.carbondata.processing.util.TableOptionConstant
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
* test case for external column dictionary generation
@@ -176,7 +176,7 @@ class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAft
CarbonCommonConstants.CARBON_DATE_FORMAT,
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
carbonLoadModel.setCsvHeaderColumns(
- CommonUtil.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
+ LoadOption.getCsvHeaderColumns(carbonLoadModel, FileFactory.getConfiguration))
carbonLoadModel.setMaxColumns("100")
// Create table and metadata folders if not exist
val metadataDirectoryPath = CarbonTablePath.getMetadataPath(table.getTablePath)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
index 7dc6275..3b599fc 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/TestStreamingTableOperation.scala
@@ -32,14 +32,12 @@ import org.apache.spark.sql.streaming.{ProcessingTime, StreamingQuery}
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.statusmanager.{FileFormat, SegmentStatus}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, ProcessMetaDataException}
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.exception.ProcessMetaDataException
class TestStreamingTableOperation extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
index 9da7244..65a006b 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala
@@ -21,11 +21,11 @@ import org.apache.spark.sql.common.util.Spark2QueryTest
import org.apache.spark.sql.execution.exchange.ShuffleExchange
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index ac10b9a..995f041 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -26,9 +26,9 @@ import org.apache.spark.sql.common.util.Spark2QueryTest
import org.apache.spark.sql.test.TestQueryExecutor
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.exception.{MalformedCarbonCommandException, ProcessMetaDataException}
class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89cfd8e0/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index 24b25b9..bf1dfd1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -18,14 +18,16 @@
package org.apache.carbondata.processing.loading.model;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import org.apache.carbondata.core.dictionary.service.DictionaryServiceProvider;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-
+import org.apache.carbondata.core.util.path.CarbonTablePath;
public class CarbonLoadModel implements Serializable {
@@ -797,6 +799,7 @@ public class CarbonLoadModel implements Serializable {
this.skipEmptyLine = skipEmptyLine;
}
+
public boolean isPartitionLoad() {
return isPartitionLoad;
}
@@ -812,4 +815,13 @@ public class CarbonLoadModel implements Serializable {
public void setDataWritePath(String dataWritePath) {
this.dataWritePath = dataWritePath;
}
+
+ /**
+ * Read segments metadata from table status file and set it to this load model object
+ */
+ public void readAndSetLoadMetadataDetails() {
+ String metadataPath = CarbonTablePath.getMetadataPath(tablePath);
+ LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(metadataPath);
+ setLoadMetadataDetails(Arrays.asList(details));
+ }
}
[17/54] [abbrv] carbondata git commit: [CARBONDATA-2186] Add
InterfaceAudience.Internal to annotate internal interface
Posted by ra...@apache.org.
[CARBONDATA-2186] Add InterfaceAudience.Internal to annotate internal interface
This closes #1986
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e72bfd1a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e72bfd1a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e72bfd1a
Branch: refs/heads/master
Commit: e72bfd1a949b9992026ebaa9df3224e4fa004422
Parents: 0539662
Author: Jacky Li <ja...@qq.com>
Authored: Tue Feb 20 11:16:53 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/carbondata/common/Maps.java | 2 +-
.../org/apache/carbondata/common/Strings.java | 2 +-
.../common/annotations/InterfaceAudience.java | 19 ++++++++++++++-----
.../common/annotations/InterfaceStability.java | 2 +-
.../loading/model/CarbonLoadModelBuilder.java | 2 +-
.../processing/loading/model/LoadOption.java | 2 +-
.../carbondata/sdk/file/CSVCarbonWriter.java | 4 +---
7 files changed, 20 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/common/src/main/java/org/apache/carbondata/common/Maps.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/Maps.java b/common/src/main/java/org/apache/carbondata/common/Maps.java
index 14fc329..4e76192 100644
--- a/common/src/main/java/org/apache/carbondata/common/Maps.java
+++ b/common/src/main/java/org/apache/carbondata/common/Maps.java
@@ -21,7 +21,7 @@ import java.util.Map;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class Maps {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/common/src/main/java/org/apache/carbondata/common/Strings.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/Strings.java b/common/src/main/java/org/apache/carbondata/common/Strings.java
index 08fdc3c..23c7f9f 100644
--- a/common/src/main/java/org/apache/carbondata/common/Strings.java
+++ b/common/src/main/java/org/apache/carbondata/common/Strings.java
@@ -21,7 +21,7 @@ import java.util.Objects;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class Strings {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
index fa9729d..8d214ff 100644
--- a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceAudience.java
@@ -25,10 +25,10 @@ import java.lang.annotation.RetentionPolicy;
* This annotation is ported and modified from Apache Hadoop project.
*
* Annotation to inform users of a package, class or method's intended audience.
- * Currently the audience can be {@link User}, {@link Developer}
+ * Currently the audience can be {@link User}, {@link Developer}, {@link Internal}
*
* Public classes that are not marked with this annotation must be
- * considered by default as {@link Developer}.</li>
+ * considered by default as {@link Internal}.</li>
*
* External applications must only use classes that are marked {@link User}.
*
@@ -47,12 +47,21 @@ public class InterfaceAudience {
public @interface User { }
/**
- * Intended only for developers to extend interface for CarbonData project
- * For example, new Datamap implementations.
+ * Intended for developers to develop extension for Apache CarbonData project
+ * For example, "Index DataMap" to add a new index implementation, etc
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
- public @interface Developer { }
+ public @interface Developer {
+ String[] value();
+ }
+
+ /**
+ * Intended only for internal usage within Apache CarbonData project.
+ */
+ @Documented
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Internal { }
private InterfaceAudience() { } // Audience can't exist on its own
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
index b8e5e52..5435028 100644
--- a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
@@ -36,7 +36,7 @@ import org.apache.carbondata.common.annotations.InterfaceAudience.*;
*
* <ul><li>All classes that are annotated with {@link User} or
* {@link Developer} must have InterfaceStability annotation. </li>
- * <li>Classes that are {@link Private} are to be considered unstable unless
+ * <li>Classes that are {@link Internal} are to be considered unstable unless
* a different InterfaceStability annotation states otherwise.</li>
* <li>Incompatible changes must not be made to classes marked as stable.</li>
* </ul>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
index fbb93b6..99684ad 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.conf.Configuration;
/**
* Builder for {@link CarbonLoadModel}
*/
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class CarbonLoadModelBuilder {
private CarbonTable table;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
index 8ec93a9..bd942ca 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/model/LoadOption.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.conf.Configuration;
/**
* Provide utilities to populate loading options
*/
-@InterfaceAudience.Developer
+@InterfaceAudience.Internal
public class LoadOption {
private static LogService LOG = LogServiceFactory.getLogService(LoadOption.class.getName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e72bfd1a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
index daea733..dc5696a 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java
@@ -22,7 +22,6 @@ import java.util.Random;
import java.util.UUID;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
@@ -40,8 +39,7 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
/**
* Implementation to write rows in CSV format to carbondata file.
*/
-@InterfaceAudience.Developer
-@InterfaceStability.Unstable
+@InterfaceAudience.Internal
class CSVCarbonWriter extends CarbonWriter {
private RecordWriter<NullWritable, StringArrayWritable> recordWriter;
[22/54] [abbrv] carbondata git commit: [CARBONDATA-2023][DataLoad]
Add size base block allocation in data loading
Posted by ra...@apache.org.
[CARBONDATA-2023][DataLoad] Add size base block allocation in data loading
Carbondata assign blocks to nodes at the beginning of data loading.
Previous block allocation strategy is block number based and it will
suffer skewed data problem if the size of input files differs a lot.
We introduced a size based block allocation strategy to optimize data
loading performance in skewed data scenario.
This closes #1808
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8d8b589e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8d8b589e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8d8b589e
Branch: refs/heads/master
Commit: 8d8b589e78a9db1ddc101d20c1e3feb500acce19
Parents: 21704cf
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Feb 8 14:42:39 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../core/datamap/dev/AbstractDataMapWriter.java | 5 +-
.../core/datamap/dev/DataMapFactory.java | 2 +-
.../blockletindex/BlockletDataMapFactory.java | 2 +-
.../SegmentUpdateStatusManager.java | 9 +-
.../carbondata/core/util/NonDictionaryUtil.java | 67 ++-
.../datamap/examples/MinMaxDataMapFactory.java | 5 +-
.../datamap/examples/MinMaxDataWriter.java | 7 +-
.../presto/util/CarbonDataStoreCreator.scala | 1 +
.../CarbonIndexFileMergeTestCase.scala | 4 -
.../testsuite/datamap/CGDataMapTestCase.scala | 26 +-
.../testsuite/datamap/DataMapWriterSuite.scala | 19 +-
.../testsuite/datamap/FGDataMapTestCase.scala | 31 +-
.../iud/DeleteCarbonTableTestCase.scala | 2 +-
.../TestInsertAndOtherCommandConcurrent.scala | 14 +-
.../StandardPartitionTableCleanTestCase.scala | 12 +-
.../StandardPartitionTableLoadingTestCase.scala | 2 +-
.../load/DataLoadProcessorStepOnSpark.scala | 6 +-
.../carbondata/spark/util/DataLoadingUtil.scala | 2 +-
.../datamap/DataMapWriterListener.java | 2 +-
.../loading/row/IntermediateSortTempRow.java | 117 -----
.../loading/sort/SortStepRowHandler.java | 466 -------------------
.../loading/sort/SortStepRowUtil.java | 103 ++++
.../sort/unsafe/UnsafeCarbonRowPage.java | 331 +++++++++++--
.../loading/sort/unsafe/UnsafeSortDataRows.java | 57 ++-
.../unsafe/comparator/UnsafeRowComparator.java | 95 ++--
.../UnsafeRowComparatorForNormalDIms.java | 59 +++
.../UnsafeRowComparatorForNormalDims.java | 59 ---
.../sort/unsafe/holder/SortTempChunkHolder.java | 3 +-
.../holder/UnsafeFinalMergePageHolder.java | 19 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 21 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 138 ++++--
.../merger/UnsafeIntermediateFileMerger.java | 118 ++++-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 27 +-
.../processing/merger/CarbonDataMergerUtil.java | 8 +-
.../merger/CompactionResultSortProcessor.java | 5 +-
.../merger/RowResultMergerProcessor.java | 5 +-
.../partition/spliter/RowResultProcessor.java | 5 +-
.../sort/sortdata/IntermediateFileMerger.java | 95 +++-
.../IntermediateSortTempRowComparator.java | 73 ---
.../sort/sortdata/NewRowComparator.java | 5 +-
.../sortdata/NewRowComparatorForNormalDims.java | 3 +-
.../processing/sort/sortdata/RowComparator.java | 94 ++++
.../sortdata/RowComparatorForNormalDims.java | 62 +++
.../SingleThreadFinalSortFilesMerger.java | 25 +-
.../processing/sort/sortdata/SortDataRows.java | 85 +++-
.../sort/sortdata/SortTempFileChunkHolder.java | 174 +++++--
.../sort/sortdata/TableFieldStat.java | 176 -------
.../util/CarbonDataProcessorUtil.java | 4 +-
.../processing/util/CarbonLoaderUtil.java | 9 -
49 files changed, 1368 insertions(+), 1291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
index bcc9bad..de6dcb1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/AbstractDataMapWriter.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datamap.dev;
import java.io.IOException;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -35,10 +36,10 @@ public abstract class AbstractDataMapWriter {
protected String writeDirectoryPath;
- public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ public AbstractDataMapWriter(AbsoluteTableIdentifier identifier, Segment segment,
String writeDirectoryPath) {
this.identifier = identifier;
- this.segmentId = segmentId;
+ this.segmentId = segment.getSegmentNo();
this.writeDirectoryPath = writeDirectoryPath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index df5670d..50ac279 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -39,7 +39,7 @@ public interface DataMapFactory<T extends DataMap> {
/**
* Return a new write for this datamap
*/
- AbstractDataMapWriter createWriter(Segment segment);
+ AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath);
/**
* Get the datamap for segmentid
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index efe2b71..ee849bd 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -72,7 +72,7 @@ public class BlockletDataMapFactory extends AbstractCoarseGrainDataMapFactory
}
@Override
- public AbstractDataMapWriter createWriter(Segment segment) {
+ public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
throw new UnsupportedOperationException("not implemented");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 94a4243..39eb262 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -260,13 +260,8 @@ public class SegmentUpdateStatusManager {
/**
* Returns all delta file paths of specified block
- *
- * @param tupleId
- * @param extension
- * @return
- * @throws Exception
*/
- public List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
+ private List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
try {
String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
String completeBlockName = CarbonTablePath.addDataPartPrefix(
@@ -405,10 +400,8 @@ public class SegmentUpdateStatusManager {
public CarbonFile[] getDeleteDeltaFilesList(final Segment segmentId, final String blockName) {
String segmentPath = CarbonTablePath.getSegmentPath(
identifier.getTablePath(), segmentId.getSegmentNo());
-
CarbonFile segDir =
FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-
for (SegmentUpdateDetails block : updateDetails) {
if ((block.getBlockName().equalsIgnoreCase(blockName)) &&
(block.getSegmentName().equalsIgnoreCase(segmentId.getSegmentNo()))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index fca1244..d6ecfbc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -82,26 +82,18 @@ public class NonDictionaryUtil {
}
/**
- * Method to get the required dictionary Dimension from obj []
+ * Method to get the required Dimension from obj []
*
* @param index
* @param row
* @return
*/
- public static int getDictDimension(int index, Object[] row) {
- int[] dimensions = (int[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+ public static Integer getDimension(int index, Object[] row) {
+
+ Integer[] dimensions = (Integer[]) row[WriteStepRowUtil.DICTIONARY_DIMENSION];
+
return dimensions[index];
- }
- /**
- * Method to get the required non-dictionary & complex from 3-parted row
- * @param index
- * @param row
- * @return
- */
- public static byte[] getNoDictOrComplex(int index, Object[] row) {
- byte[][] nonDictArray = (byte[][]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
- return nonDictArray[index];
}
/**
@@ -116,11 +108,60 @@ public class NonDictionaryUtil {
return measures[index];
}
+ public static byte[] getByteArrayForNoDictionaryCols(Object[] row) {
+
+ return (byte[]) row[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX];
+ }
+
public static void prepareOutObj(Object[] out, int[] dimArray, byte[][] byteBufferArr,
Object[] measureArray) {
+
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
out[WriteStepRowUtil.MEASURE] = measureArray;
}
+
+ /**
+ * This method will extract the single dimension from the complete high card dims byte[].+ *
+ * The format of the byte [] will be, Totallength,CompleteStartOffsets,Dat
+ *
+ * @param highCardArr
+ * @param index
+ * @param highCardinalityCount
+ * @param outBuffer
+ */
+ public static void extractSingleHighCardDims(byte[] highCardArr, int index,
+ int highCardinalityCount, ByteBuffer outBuffer) {
+ ByteBuffer buff = null;
+ short secIndex = 0;
+ short firstIndex = 0;
+ int length;
+ // if the requested index is a last one then we need to calculate length
+ // based on byte[] length.
+ if (index == highCardinalityCount - 1) {
+ // need to read 2 bytes(1 short) to determine starting offset and
+ // length can be calculated by array length.
+ buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 2);
+ } else {
+ // need to read 4 bytes(2 short) to determine starting offset and
+ // length.
+ buff = ByteBuffer.wrap(highCardArr, (index * 2) + 2, 4);
+ }
+
+ firstIndex = buff.getShort();
+ // if it is a last dimension in high card then this will be last
+ // offset.so calculate length from total length
+ if (index == highCardinalityCount - 1) {
+ secIndex = (short) highCardArr.length;
+ } else {
+ secIndex = buff.getShort();
+ }
+
+ length = secIndex - firstIndex;
+
+ outBuffer.position(firstIndex);
+ outBuffer.limit(outBuffer.position() + length);
+
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
index 266c107..4ef74a7 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
@@ -52,9 +52,8 @@ public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
* @param segment
* @return
*/
- @Override public AbstractDataMapWriter createWriter(Segment segment) {
- return new MinMaxDataWriter(identifier, segment.getSegmentNo(),
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
+ @Override public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
+ return new MinMaxDataWriter(identifier, segment, writeDirectoryPath);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index fe0bbcf..5046182 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -52,11 +53,11 @@ public class MinMaxDataWriter extends AbstractDataMapWriter {
private String dataWritePath;
- public MinMaxDataWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ public MinMaxDataWriter(AbsoluteTableIdentifier identifier, Segment segment,
String dataWritePath) {
- super(identifier, segmentId, dataWritePath);
+ super(identifier, segment, dataWritePath);
this.identifier = identifier;
- this.segmentId = segmentId;
+ this.segmentId = segment.getSegmentNo();
this.dataWritePath = dataWritePath;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index 1bc9812..e768660 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -383,6 +383,7 @@ object CarbonDataStoreCreator {
.getInstance.createCache(CacheType.REVERSE_DICTIONARY)
for (i <- set.indices) {
+ // val dim = getDimension(dims, i).get
val columnIdentifier: ColumnIdentifier =
new ColumnIdentifier(dims.get(i).getColumnId, null, null)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index 7608318..aace3ea 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -85,7 +85,6 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
- val carbonTablePath = new CarbonTablePath(table.getCarbonTableIdentifier, table.getTablePath)
new CarbonIndexFileMergeWriter()
.mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
new CarbonIndexFileMergeWriter()
@@ -111,7 +110,6 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
- val carbonTablePath = new CarbonTablePath(table.getCarbonTableIdentifier, table.getTablePath)
new CarbonIndexFileMergeWriter()
.mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
new CarbonIndexFileMergeWriter()
@@ -141,7 +139,6 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
- val carbonTablePath = new CarbonTablePath(table.getCarbonTableIdentifier, table.getTablePath)
new CarbonIndexFileMergeWriter()
.mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default_nonindexmerge", "0.1") == 0)
@@ -171,7 +168,6 @@ class CarbonIndexFileMergeTestCase
assert(getIndexFileCount("default_nonindexmerge", "3") == 100)
sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
val table = CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
- val carbonTablePath = new CarbonTablePath(table.getCarbonTableIdentifier, table.getTablePath)
new CarbonIndexFileMergeWriter()
.mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index 4b6f231..1cbbcb4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -27,14 +27,14 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
import org.apache.carbondata.core.datastore.FileReader
import org.apache.carbondata.core.datastore.block.SegmentProperties
import org.apache.carbondata.core.datastore.compression.SnappyCompressor
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.Blocklet
+import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.scan.expression.Expression
@@ -62,16 +62,16 @@ class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
/**
* Return a new write for this datamap
*/
- override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
- new CGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
+ new CGDataMapWriter(identifier, segment, dataWritePath, dataMapName)
}
/**
* Get the datamap for segmentid
*/
- override def getDataMaps(segmentId: String): java.util.List[AbstractCoarseGrainDataMap] = {
+ override def getDataMaps(segment: Segment): java.util.List[AbstractCoarseGrainDataMap] = {
val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
val files = file.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
@@ -108,9 +108,9 @@ class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
*
* @return
*/
- override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
val file = FileFactory.getCarbonFile(
- CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
val files = file.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
@@ -125,7 +125,7 @@ class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
/**
* Clears datamap of the segment
*/
- override def clear(segmentId: String): Unit = {
+ override def clear(segment: Segment): Unit = {
}
@@ -175,7 +175,7 @@ class CGDataMap extends AbstractCoarseGrainDataMap {
override def prune(
filterExp: FilterResolverIntf,
segmentProperties: SegmentProperties,
- partitions: java.util.List[String]): java.util.List[Blocklet] = {
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
val expression = filterExp.getFilterExpression
getEqualToExpression(expression, buffer)
@@ -184,7 +184,7 @@ class CGDataMap extends AbstractCoarseGrainDataMap {
}
val meta = findMeta(value(0).getBytes)
meta.map { f=>
- new Blocklet(f._1, f._2+"")
+ new Blocklet(f._1, f._2 + "")
}.asJava
}
@@ -219,10 +219,10 @@ class CGDataMap extends AbstractCoarseGrainDataMap {
}
class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segmentId: String,
+ segment: Segment,
dataWritePath: String,
dataMapName: String)
- extends AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
+ extends AbstractDataMapWriter(identifier, segment, dataWritePath) {
var currentBlockId: String = null
val cgwritepath = dataWritePath + "/" +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 2f8a1d1..7e93959 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -20,21 +20,19 @@ package org.apache.carbondata.spark.testsuite.datamap
import java.util
import scala.collection.JavaConverters._
+
import org.apache.spark.sql.test.util.QueryTest
import org.apache.spark.sql.{DataFrame, SaveMode}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datamap.dev.{DataMap, DataMapFactory, DataMapWriter}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter
+import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMap}
import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainDataMap, AbstractCoarseGrainDataMapFactory}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
-import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
@@ -49,15 +47,16 @@ class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
override def fireEvent(event: Event): Unit = ???
- override def clear(segmentId: Segment): Unit = {}
+ override def clear(segment: Segment): Unit = {}
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: Segment): util.List[DataMap] = ???
+ override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: Segment): AbstractDataMapWriter = DataMapWriterSuite.dataMapWriterC2Mock
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter =
+ DataMapWriterSuite.dataMapWriterC2Mock(identifier, segment, dataWritePath)
override def getMeta: DataMapMeta = new DataMapMeta(List("c2").asJava, List(ExpressionType.EQUALS).asJava)
@@ -175,9 +174,9 @@ object DataMapWriterSuite {
var callbackSeq: Seq[String] = Seq[String]()
- def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segmentId: String,
+ def dataMapWriterC2Mock(identifier: AbsoluteTableIdentifier, segment: Segment,
dataWritePath: String) =
- new AbstractDataMapWriter(identifier, segmentId, dataWritePath) {
+ new AbstractDataMapWriter(identifier, segment, dataWritePath) {
override def onPageAdded(
blockletId: Int,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index d1bb65f..9c8cc15 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -27,14 +27,14 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.datamap.dev.fgdatamap.{AbstractFineGrainDataMap, AbstractFineGrainDataMapFactory}
import org.apache.carbondata.core.datamap.dev.{AbstractDataMapWriter, DataMapModel}
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager}
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, DataMapStoreManager, Segment}
import org.apache.carbondata.core.datastore.FileReader
import org.apache.carbondata.core.datastore.block.SegmentProperties
import org.apache.carbondata.core.datastore.compression.SnappyCompressor
import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.page.ColumnPage
-import org.apache.carbondata.core.indexstore.FineGrainBlocklet
+import org.apache.carbondata.core.indexstore.{Blocklet, FineGrainBlocklet, PartitionSpec}
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.scan.expression.Expression
@@ -62,16 +62,16 @@ class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
/**
* Return a new write for this datamap
*/
- override def createWriter(segmentId: String, dataWritePath: String): AbstractDataMapWriter = {
- new FGDataMapWriter(identifier, segmentId, dataWritePath, dataMapName)
+ override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
+ new FGDataMapWriter(identifier, segment, dataWritePath, dataMapName)
}
/**
* Get the datamap for segmentid
*/
- override def getDataMaps(segmentId: String): java.util.List[AbstractFineGrainDataMap] = {
+ override def getDataMaps(segment: Segment): java.util.List[AbstractFineGrainDataMap] = {
val file = FileFactory
- .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+ .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
val files = file.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
@@ -99,9 +99,9 @@ class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
*
* @return
*/
- override def toDistributable(segmentId: String): java.util.List[DataMapDistributable] = {
- val file = FileFactory
- .getCarbonFile(CarbonTablePath.getSegmentPath(identifier.getTablePath, segmentId))
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
val files = file.listFiles(new CarbonFileFilter {
override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
@@ -112,7 +112,6 @@ class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
}.toList.asJava
}
-
/**
*
* @param event
@@ -124,7 +123,7 @@ class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
/**
* Clears datamap of the segment
*/
- override def clear(segmentId: String): Unit = {
+ override def clear(segment: Segment): Unit = {
}
/**
@@ -173,7 +172,7 @@ class FGDataMap extends AbstractFineGrainDataMap {
override def prune(
filterExp: FilterResolverIntf,
segmentProperties: SegmentProperties,
- partitions: java.util.List[String]): java.util.List[FineGrainBlocklet] = {
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
val expression = filterExp.getFilterExpression
getEqualToExpression(expression, buffer)
@@ -187,7 +186,7 @@ class FGDataMap extends AbstractFineGrainDataMap {
}
private def readAndFindData(meta: (String, Int, (Array[Byte], Array[Byte]), Long, Int),
- value: Array[Byte]): Option[FineGrainBlocklet] = {
+ value: Array[Byte]): Option[Blocklet] = {
val bytes = FileReader.readByteArray(filePath, meta._4, meta._5)
val outputStream = new ByteArrayInputStream(compressor.unCompressByte(bytes))
val obj = new ObjectInputStream(outputStream)
@@ -211,12 +210,10 @@ class FGDataMap extends AbstractFineGrainDataMap {
pg.setRowId(f._2(p._2).toArray)
pg
}
- pages
Some(new FineGrainBlocklet(meta._1, meta._2.toString, pages.toList.asJava))
} else {
None
}
-
}
private def findMeta(value: Array[Byte]) = {
@@ -249,8 +246,8 @@ class FGDataMap extends AbstractFineGrainDataMap {
}
class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segmentId: String, dataWriterPath: String, dataMapName: String)
- extends AbstractDataMapWriter(identifier, segmentId, dataWriterPath) {
+ segment: Segment, dataWriterPath: String, dataMapName: String)
+ extends AbstractDataMapWriter(identifier, segment, dataWriterPath) {
var currentBlockId: String = null
val fgwritepath = dataWriterPath + "/" + System.nanoTime() + ".datamap"
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index d05f022..510903a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -194,7 +194,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("delete from update_status_files where age=5").show()
val carbonTable = CarbonEnv
.getCarbonTable(Some("iud_db"), "update_status_files")(sqlContext.sparkSession)
- val metaPath = carbonTable.getMetaDataFilepath
+ val metaPath = carbonTable.getMetadataPath
val files = FileFactory.getCarbonFile(metaPath)
val result = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore.getClass
if(result.getCanonicalName.contains("CarbonFileMetastore")) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index 5550358..b39c44c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -269,7 +269,11 @@ object Global {
class WaitingDataMap() extends AbstractCoarseGrainDataMapFactory {
- override def init(identifier: AbsoluteTableIdentifier, dataMapName: String): Unit = { }
+ private var identifier: AbsoluteTableIdentifier = _
+
+ override def init(identifier: AbsoluteTableIdentifier, dataMapName: String): Unit = {
+ this.identifier = identifier
+ }
override def fireEvent(event: Event): Unit = ???
@@ -277,12 +281,12 @@ class WaitingDataMap() extends AbstractCoarseGrainDataMapFactory {
override def clear(): Unit = {}
- override def getDataMaps(distributable: DataMapDistributable): java.util.List[AbstractCoarseGrainDataMap] = ???
+ override def getDataMaps(distributable: DataMapDistributable): util.List[AbstractCoarseGrainDataMap] = ???
- override def getDataMaps(segmentId: Segment): util.List[DataMap] = ???
+ override def getDataMaps(segment: Segment): util.List[AbstractCoarseGrainDataMap] = ???
- override def createWriter(segmentId: Segment): AbstractDataMapWriter = {
- new AbstractDataMapWriter {
+ override def createWriter(segment: Segment, writeDirectoryPath: String): AbstractDataMapWriter = {
+ new AbstractDataMapWriter(identifier, segment, writeDirectoryPath) {
override def onPageAdded(blockletId: Int, pageId: Int, pages: Array[ColumnPage]): Unit = { }
override def onBlockletEnd(blockletId: Int): Unit = { }
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
index f238d2b..cfc6983 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableCleanTestCase.scala
@@ -52,14 +52,12 @@ class StandardPartitionTableCleanTestCase extends QueryTest with BeforeAndAfterA
def validateDataFiles(tableUniqueName: String, segmentId: String, partition: Int, indexes: Int): Unit = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- val tablePath = new CarbonTablePath(carbonTable.getCarbonTableIdentifier,
- carbonTable.getTablePath)
- val partitions = CarbonFilters
- .getPartitions(Seq.empty,
- sqlContext.sparkSession,
- TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
+ val partitions = CarbonFilters.getPartitions(
+ Seq.empty,
+ sqlContext.sparkSession,
+ TableIdentifier(carbonTable.getTableName, Some(carbonTable.getDatabaseName)))
assert(partitions.get.length == partition)
- val details = SegmentStatusManager.readLoadMetadata(tablePath.getMetadataDirectoryPath)
+ val details = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(carbonTable.getTablePath))
val segLoad = details.find(_.getLoadName.equals(segmentId)).get
val seg = new SegmentFileStore(carbonTable.getTablePath, segLoad.getSegmentFile)
assert(seg.getIndexFiles.size == indexes)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
index 2ce46ef..baf1627 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/standardpartition/StandardPartitionTableLoadingTestCase.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.spark.rdd.CarbonScanRDD
class StandardPartitionTableLoadingTestCase extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
index 0422239..5124247 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/load/DataLoadProcessorStepOnSpark.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl
import org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.loading.model.CarbonLoadModel
import org.apache.carbondata.processing.loading.parser.impl.RowParserImpl
-import org.apache.carbondata.processing.loading.sort.SortStepRowHandler
+import org.apache.carbondata.processing.loading.sort.SortStepRowUtil
import org.apache.carbondata.processing.loading.steps.DataWriterProcessorStepImpl
import org.apache.carbondata.processing.sort.sortdata.SortParameters
import org.apache.carbondata.processing.store.{CarbonFactHandler, CarbonFactHandlerFactory}
@@ -206,7 +206,7 @@ object DataLoadProcessorStepOnSpark {
val model: CarbonLoadModel = modelBroadcast.value.getCopyWithTaskNo(index.toString)
val conf = DataLoadProcessBuilder.createConfiguration(model)
val sortParameters = SortParameters.createSortParameters(conf)
- val sortStepRowHandler = new SortStepRowHandler(sortParameters)
+ val sortStepRowUtil = new SortStepRowUtil(sortParameters)
TaskContext.get().addTaskFailureListener { (t: TaskContext, e: Throwable) =>
wrapException(e, model)
}
@@ -216,7 +216,7 @@ object DataLoadProcessorStepOnSpark {
override def next(): CarbonRow = {
val row =
- new CarbonRow(sortStepRowHandler.convertRawRowTo3Parts(rows.next().getData))
+ new CarbonRow(sortStepRowUtil.convertRow(rows.next().getData))
rowCounter.add(1)
row
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index cee40c8..49e4420 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -441,7 +441,7 @@ object DataLoadingUtil {
private def isUpdationRequired(isForceDeletion: Boolean,
carbonTable: CarbonTable,
- absoluteTableIdentifier: AbsoluteTableIdentifier) = {
+ absoluteTableIdentifier: AbsoluteTableIdentifier): (Array[LoadMetadataDetails], Boolean) = {
val details = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath)
// Delete marked loads
val isUpdationRequired =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 5083ab5..1104229 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -74,7 +74,7 @@ public class DataMapWriterListener {
}
List<String> columns = factory.getMeta().getIndexedColumns();
List<AbstractDataMapWriter> writers = registry.get(columns);
- AbstractDataMapWriter writer = factory.createWriter(new Segment(segmentId, null));
+ AbstractDataMapWriter writer = factory.createWriter(new Segment(segmentId, null), dataWritePath);
if (writers != null) {
writers.add(writer);
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java b/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
deleted file mode 100644
index 8d351cf..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/row/IntermediateSortTempRow.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.loading.row;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.DataTypeUtil;
-
-/**
- * During sort procedure, each row will be written to sort temp file in this logic format.
- * an intermediate sort temp row consists 3 parts:
- * dictSort, noDictSort, noSortDimsAndMeasures(dictNoSort, noDictNoSort, measure)
- */
-public class IntermediateSortTempRow {
- private int[] dictSortDims;
- private byte[][] noDictSortDims;
- private byte[] noSortDimsAndMeasures;
-
- public IntermediateSortTempRow(int[] dictSortDims, byte[][] noDictSortDims,
- byte[] noSortDimsAndMeasures) {
- this.dictSortDims = dictSortDims;
- this.noDictSortDims = noDictSortDims;
- this.noSortDimsAndMeasures = noSortDimsAndMeasures;
- }
-
- public int[] getDictSortDims() {
- return dictSortDims;
- }
-
- public byte[][] getNoDictSortDims() {
- return noDictSortDims;
- }
-
- public byte[] getNoSortDimsAndMeasures() {
- return noSortDimsAndMeasures;
- }
-
- /**
- * deserialize from bytes array to get the no sort fields
- * @param outDictNoSort stores the dict & no-sort fields
- * @param outNoDictNoSort stores the no-dict & no-sort fields, including complex
- * @param outMeasures stores the measure fields
- * @param dataTypes data type for the measure
- */
- public void unpackNoSortFromBytes(int[] outDictNoSort, byte[][] outNoDictNoSort,
- Object[] outMeasures, DataType[] dataTypes) {
- ByteBuffer rowBuffer = ByteBuffer.wrap(noSortDimsAndMeasures);
- // read dict_no_sort
- int dictNoSortCnt = outDictNoSort.length;
- for (int i = 0; i < dictNoSortCnt; i++) {
- outDictNoSort[i] = rowBuffer.getInt();
- }
-
- // read no_dict_no_sort (including complex)
- int noDictNoSortCnt = outNoDictNoSort.length;
- for (int i = 0; i < noDictNoSortCnt; i++) {
- short len = rowBuffer.getShort();
- byte[] bytes = new byte[len];
- rowBuffer.get(bytes);
- outNoDictNoSort[i] = bytes;
- }
-
- // read measure
- int measureCnt = outMeasures.length;
- DataType tmpDataType;
- Object tmpContent;
- for (short idx = 0 ; idx < measureCnt; idx++) {
- if ((byte) 0 == rowBuffer.get()) {
- outMeasures[idx] = null;
- continue;
- }
-
- tmpDataType = dataTypes[idx];
- if (DataTypes.BOOLEAN == tmpDataType) {
- if ((byte) 1 == rowBuffer.get()) {
- tmpContent = true;
- } else {
- tmpContent = false;
- }
- } else if (DataTypes.SHORT == tmpDataType) {
- tmpContent = rowBuffer.getShort();
- } else if (DataTypes.INT == tmpDataType) {
- tmpContent = rowBuffer.getInt();
- } else if (DataTypes.LONG == tmpDataType) {
- tmpContent = rowBuffer.getLong();
- } else if (DataTypes.DOUBLE == tmpDataType) {
- tmpContent = rowBuffer.getDouble();
- } else if (DataTypes.isDecimal(tmpDataType)) {
- short len = rowBuffer.getShort();
- byte[] decimalBytes = new byte[len];
- rowBuffer.get(decimalBytes);
- tmpContent = DataTypeUtil.byteToBigDecimal(decimalBytes);
- } else {
- throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
- }
- outMeasures[idx] = tmpContent;
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
deleted file mode 100644
index f31a2b9..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowHandler.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.processing.loading.sort;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.NonDictionaryUtil;
-import org.apache.carbondata.processing.loading.row.IntermediateSortTempRow;
-import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.carbondata.processing.sort.sortdata.TableFieldStat;
-
-/**
- * This class is used to convert/write/read row in sort step in carbondata.
- * It consists the following function:
- * 1. convert raw row & intermediate sort temp row to 3-parted row
- * 2. read/write intermediate sort temp row to sort temp file & unsafe memory
- * 3. write raw row directly to sort temp file & unsafe memory as intermediate sort temp row
- */
-public class SortStepRowHandler implements Serializable {
- private static final long serialVersionUID = 1L;
- private int dictSortDimCnt = 0;
- private int dictNoSortDimCnt = 0;
- private int noDictSortDimCnt = 0;
- private int noDictNoSortDimCnt = 0;
- private int measureCnt;
-
- // indices for dict & sort dimension columns
- private int[] dictSortDimIdx;
- // indices for dict & no-sort dimension columns
- private int[] dictNoSortDimIdx;
- // indices for no-dict & sort dimension columns
- private int[] noDictSortDimIdx;
- // indices for no-dict & no-sort dimension columns, including complex columns
- private int[] noDictNoSortDimIdx;
- // indices for measure columns
- private int[] measureIdx;
-
- private DataType[] dataTypes;
-
- /**
- * constructor
- * @param tableFieldStat table field stat
- */
- public SortStepRowHandler(TableFieldStat tableFieldStat) {
- this.dictSortDimCnt = tableFieldStat.getDictSortDimCnt();
- this.dictNoSortDimCnt = tableFieldStat.getDictNoSortDimCnt();
- this.noDictSortDimCnt = tableFieldStat.getNoDictSortDimCnt();
- this.noDictNoSortDimCnt = tableFieldStat.getNoDictNoSortDimCnt();
- this.measureCnt = tableFieldStat.getMeasureCnt();
- this.dictSortDimIdx = tableFieldStat.getDictSortDimIdx();
- this.dictNoSortDimIdx = tableFieldStat.getDictNoSortDimIdx();
- this.noDictSortDimIdx = tableFieldStat.getNoDictSortDimIdx();
- this.noDictNoSortDimIdx = tableFieldStat.getNoDictNoSortDimIdx();
- this.measureIdx = tableFieldStat.getMeasureIdx();
- this.dataTypes = tableFieldStat.getMeasureDataType();
- }
-
- /**
- * constructor
- * @param sortParameters sort parameters
- */
- public SortStepRowHandler(SortParameters sortParameters) {
- this(new TableFieldStat(sortParameters));
- }
-
- /**
- * Convert carbon row from raw format to 3-parted format.
- * This method is used in global-sort.
- *
- * @param row raw row whose length is the same as field number
- * @return 3-parted row whose length is 3. (1 for dict dims ,1 for non-dict and complex,
- * 1 for measures)
- */
- public Object[] convertRawRowTo3Parts(Object[] row) {
- Object[] holder = new Object[3];
- try {
- int[] dictDims
- = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
- byte[][] nonDictArray = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
- Object[] measures = new Object[this.measureCnt];
-
- // convert dict & data
- int idxAcc = 0;
- for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
- dictDims[idxAcc++] = (int) row[this.dictSortDimIdx[idx]];
- }
-
- // convert dict & no-sort
- for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
- dictDims[idxAcc++] = (int) row[this.dictNoSortDimIdx[idx]];
- }
- // convert no-dict & sort
- idxAcc = 0;
- for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
- nonDictArray[idxAcc++] = (byte[]) row[this.noDictSortDimIdx[idx]];
- }
- // convert no-dict & no-sort
- for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
- nonDictArray[idxAcc++] = (byte[]) row[this.noDictNoSortDimIdx[idx]];
- }
-
- // convert measure data
- for (int idx = 0; idx < this.measureCnt; idx++) {
- measures[idx] = row[this.measureIdx[idx]];
- }
-
- NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
- } catch (Exception e) {
- throw new RuntimeException("Problem while converting row to 3 parts", e);
- }
- return holder;
- }
-
- /**
- * Convert intermediate sort temp row to 3-parted row.
- * This method is used in the final merge sort to feed rows to the next write step.
- *
- * @param sortTempRow intermediate sort temp row
- * @return 3-parted row
- */
- public Object[] convertIntermediateSortTempRowTo3Parted(IntermediateSortTempRow sortTempRow) {
- int[] dictDims
- = new int[this.dictSortDimCnt + this.dictNoSortDimCnt];
- byte[][] noDictArray
- = new byte[this.noDictSortDimCnt + this.noDictNoSortDimCnt][];
-
- int[] dictNoSortDims = new int[this.dictNoSortDimCnt];
- byte[][] noDictNoSortDims = new byte[this.noDictNoSortDimCnt][];
- Object[] measures = new Object[this.measureCnt];
-
- sortTempRow.unpackNoSortFromBytes(dictNoSortDims, noDictNoSortDims, measures, this.dataTypes);
-
- // dict dims
- System.arraycopy(sortTempRow.getDictSortDims(), 0 , dictDims,
- 0, this.dictSortDimCnt);
- System.arraycopy(dictNoSortDims, 0, dictDims,
- this.dictSortDimCnt, this.dictNoSortDimCnt);;
-
- // no dict dims, including complex
- System.arraycopy(sortTempRow.getNoDictSortDims(), 0,
- noDictArray, 0, this.noDictSortDimCnt);
- System.arraycopy(noDictNoSortDims, 0, noDictArray,
- this.noDictSortDimCnt, this.noDictNoSortDimCnt);
-
- // measures are already here
-
- Object[] holder = new Object[3];
- NonDictionaryUtil.prepareOutObj(holder, dictDims, noDictArray, measures);
- return holder;
- }
-
- /**
- * Read intermediate sort temp row from InputStream.
- * This method is used during the merge sort phase to read row from sort temp file.
- *
- * @param inputStream input stream
- * @return a row that contains three parts
- * @throws IOException if error occrus while reading from stream
- */
- public IntermediateSortTempRow readIntermediateSortTempRowFromInputStream(
- DataInputStream inputStream) throws IOException {
- int[] dictSortDims = new int[this.dictSortDimCnt];
- byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
-
- // read dict & sort dim data
- for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
- dictSortDims[idx] = inputStream.readInt();
- }
-
- // read no-dict & sort data
- for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
- short len = inputStream.readShort();
- byte[] bytes = new byte[len];
- inputStream.readFully(bytes);
- noDictSortDims[idx] = bytes;
- }
-
- // read no-dict dims & measures
- int len = inputStream.readInt();
- byte[] noSortDimsAndMeasures = new byte[len];
- inputStream.readFully(noSortDimsAndMeasures);
-
- return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
- }
-
- /**
- * Write intermediate sort temp row to OutputStream
- * This method is used during the merge sort phase to write row to sort temp file.
- *
- * @param sortTempRow intermediate sort temp row
- * @param outputStream output stream
- * @throws IOException if error occurs while writing to stream
- */
- public void writeIntermediateSortTempRowToOutputStream(IntermediateSortTempRow sortTempRow,
- DataOutputStream outputStream) throws IOException {
- // write dict & sort dim
- for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
- outputStream.writeInt(sortTempRow.getDictSortDims()[idx]);
- }
-
- // write no-dict & sort dim
- for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
- byte[] bytes = sortTempRow.getNoDictSortDims()[idx];
- outputStream.writeShort(bytes.length);
- outputStream.write(bytes);
- }
-
- // write packed no-sort dim & measure
- outputStream.writeInt(sortTempRow.getNoSortDimsAndMeasures().length);
- outputStream.write(sortTempRow.getNoSortDimsAndMeasures());
- }
-
- /**
- * Write raw row as an intermediate sort temp row to sort temp file.
- * This method is used in the beginning of the sort phase. Comparing with converting raw row to
- * intermediate sort temp row and then writing the converted one, Writing raw row directly will
- * save the intermediate trivial loss.
- * This method use an array backend buffer to save memory allocation. The buffer will be reused
- * for all rows (per thread).
- *
- * @param row raw row
- * @param outputStream output stream
- * @param rowBuffer array backend buffer
- * @throws IOException if error occurs while writing to stream
- */
- public void writeRawRowAsIntermediateSortTempRowToOutputStream(Object[] row,
- DataOutputStream outputStream, ByteBuffer rowBuffer) throws IOException {
- // write dict & sort
- for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
- outputStream.writeInt((int) row[this.dictSortDimIdx[idx]]);
- }
-
- // write no-dict & sort
- for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
- byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
- outputStream.writeShort(bytes.length);
- outputStream.write(bytes);
- }
-
- // pack no-sort
- rowBuffer.clear();
- packNoSortFieldsToBytes(row, rowBuffer);
- rowBuffer.flip();
- int packSize = rowBuffer.limit();
-
- // write no-sort
- outputStream.writeInt(packSize);
- outputStream.write(rowBuffer.array(), 0, packSize);
- }
-
- /**
- * Read intermediate sort temp row from unsafe memory.
- * This method is used during merge sort phase for off-heap sort.
- *
- * @param baseObject base object of memory block
- * @param address address of the row
- * @return intermediate sort temp row
- */
- public IntermediateSortTempRow readIntermediateSortTempRowFromUnsafeMemory(Object baseObject,
- long address) {
- int size = 0;
-
- int[] dictSortDims = new int[this.dictSortDimCnt];
- byte[][] noDictSortDims = new byte[this.noDictSortDimCnt][];
-
- // read dict & sort dim
- for (int idx = 0; idx < dictSortDims.length; idx++) {
- dictSortDims[idx] = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- }
-
- // read no-dict & sort dim
- for (int idx = 0; idx < noDictSortDims.length; idx++) {
- short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- byte[] bytes = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
- bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
- size += length;
- noDictSortDims[idx] = bytes;
- }
-
- // read no-sort dims & measures
- int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- byte[] noSortDimsAndMeasures = new byte[len];
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
- noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
-
- return new IntermediateSortTempRow(dictSortDims, noDictSortDims, noSortDimsAndMeasures);
- }
-
- /**
- * Write intermediate sort temp row directly from unsafe memory to stream.
- * This method is used at the late beginning of the sort phase to write in-memory pages
- * to sort temp file. Comparing with reading intermediate sort temp row from memory and then
- * writing it, Writing directly from memory to stream will save the intermediate trivial loss.
- *
- * @param baseObject base object of the memory block
- * @param address base address of the row
- * @param outputStream output stream
- * @throws IOException if error occurs while writing to stream
- */
- public void writeIntermediateSortTempRowFromUnsafeMemoryToStream(Object baseObject,
- long address, DataOutputStream outputStream) throws IOException {
- int size = 0;
-
- // dict & sort
- for (int idx = 0; idx < dictSortDimCnt; idx++) {
- outputStream.writeInt(CarbonUnsafe.getUnsafe().getInt(baseObject, address + size));
- size += 4;
- }
-
- // no-dict & sort
- for (int idx = 0; idx < noDictSortDimCnt; idx++) {
- short length = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
- size += 2;
- byte[] bytes = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
- bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
- size += length;
-
- outputStream.writeShort(length);
- outputStream.write(bytes);
- }
-
- // packed no-sort & measure
- int len = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
- size += 4;
- byte[] noSortDimsAndMeasures = new byte[len];
- CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size,
- noSortDimsAndMeasures, CarbonUnsafe.BYTE_ARRAY_OFFSET, len);
- size += len;
-
- outputStream.writeInt(len);
- outputStream.write(noSortDimsAndMeasures);
- }
-
- /**
- * Write raw row as an intermediate sort temp row to memory.
- * This method is used in the beginning of the off-heap sort phase. Comparing with converting
- * raw row to intermediate sort temp row and then writing the converted one,
- * Writing raw row directly will save the intermediate trivial loss.
- * This method use an array backend buffer to save memory allocation. The buffer will be reused
- * for all rows (per thread).
- *
- * @param row raw row
- * @param baseObject base object of the memory block
- * @param address base address for the row
- * @param rowBuffer array backend buffer
- * @return number of bytes written to memory
- */
- public int writeRawRowAsIntermediateSortTempRowToUnsafeMemory(Object[] row,
- Object baseObject, long address, ByteBuffer rowBuffer) {
- int size = 0;
- // write dict & sort
- for (int idx = 0; idx < this.dictSortDimCnt; idx++) {
- CarbonUnsafe.getUnsafe()
- .putInt(baseObject, address + size, (int) row[this.dictSortDimIdx[idx]]);
- size += 4;
- }
-
- // write no-dict & sort
- for (int idx = 0; idx < this.noDictSortDimCnt; idx++) {
- byte[] bytes = (byte[]) row[this.noDictSortDimIdx[idx]];
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) bytes.length);
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
- bytes.length);
- size += bytes.length;
- }
-
- // convert pack no-sort
- rowBuffer.clear();
- packNoSortFieldsToBytes(row, rowBuffer);
- rowBuffer.flip();
- int packSize = rowBuffer.limit();
-
- // write no-sort
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, packSize);
- size += 4;
- CarbonUnsafe.getUnsafe()
- .copyMemory(rowBuffer.array(), CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
- packSize);
- size += packSize;
- return size;
- }
-
- /**
- * Pack to no-sort fields to byte array
- *
- * @param row raw row
- * @param rowBuffer byte array backend buffer
- */
- private void packNoSortFieldsToBytes(Object[] row, ByteBuffer rowBuffer) {
- // convert dict & no-sort
- for (int idx = 0; idx < this.dictNoSortDimCnt; idx++) {
- rowBuffer.putInt((int) row[this.dictNoSortDimIdx[idx]]);
- }
- // convert no-dict & no-sort
- for (int idx = 0; idx < this.noDictNoSortDimCnt; idx++) {
- byte[] bytes = (byte[]) row[this.noDictNoSortDimIdx[idx]];
- rowBuffer.putShort((short) bytes.length);
- rowBuffer.put(bytes);
- }
-
- // convert measure
- Object tmpValue;
- DataType tmpDataType;
- for (int idx = 0; idx < this.measureCnt; idx++) {
- tmpValue = row[this.measureIdx[idx]];
- tmpDataType = this.dataTypes[idx];
- if (null == tmpValue) {
- rowBuffer.put((byte) 0);
- continue;
- }
- rowBuffer.put((byte) 1);
- if (DataTypes.BOOLEAN == tmpDataType) {
- if ((boolean) tmpValue) {
- rowBuffer.put((byte) 1);
- } else {
- rowBuffer.put((byte) 0);
- }
- } else if (DataTypes.SHORT == tmpDataType) {
- rowBuffer.putShort((Short) tmpValue);
- } else if (DataTypes.INT == tmpDataType) {
- rowBuffer.putInt((Integer) tmpValue);
- } else if (DataTypes.LONG == tmpDataType) {
- rowBuffer.putLong((Long) tmpValue);
- } else if (DataTypes.DOUBLE == tmpDataType) {
- rowBuffer.putDouble((Double) tmpValue);
- } else if (DataTypes.isDecimal(tmpDataType)) {
- byte[] decimalBytes = DataTypeUtil.bigDecimalToByte((BigDecimal) tmpValue);
- rowBuffer.putShort((short) decimalBytes.length);
- rowBuffer.put(decimalBytes);
- } else {
- throw new IllegalArgumentException("Unsupported data type: " + tmpDataType);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d8b589e/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
new file mode 100644
index 0000000..c4e4756
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/SortStepRowUtil.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.loading.sort;
+
+import org.apache.carbondata.core.util.NonDictionaryUtil;
+import org.apache.carbondata.processing.sort.sortdata.SortParameters;
+
+public class SortStepRowUtil {
+ private int measureCount;
+ private int dimensionCount;
+ private int complexDimensionCount;
+ private int noDictionaryCount;
+ private int[] dictDimIdx;
+ private int[] nonDictIdx;
+ private int[] measureIdx;
+
+ public SortStepRowUtil(SortParameters parameters) {
+ this.measureCount = parameters.getMeasureColCount();
+ this.dimensionCount = parameters.getDimColCount();
+ this.complexDimensionCount = parameters.getComplexDimColCount();
+ this.noDictionaryCount = parameters.getNoDictionaryCount();
+ boolean[] isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
+
+ int index = 0;
+ int nonDicIndex = 0;
+ int allCount = 0;
+
+ // be careful that the default value is 0
+ this.dictDimIdx = new int[dimensionCount - noDictionaryCount];
+ this.nonDictIdx = new int[noDictionaryCount + complexDimensionCount];
+ this.measureIdx = new int[measureCount];
+
+ // indices for dict dim columns
+ for (int i = 0; i < isNoDictionaryDimensionColumn.length; i++) {
+ if (isNoDictionaryDimensionColumn[i]) {
+ nonDictIdx[nonDicIndex++] = i;
+ } else {
+ dictDimIdx[index++] = allCount;
+ }
+ allCount++;
+ }
+
+ // indices for non dict dim/complex columns
+ for (int i = 0; i < complexDimensionCount; i++) {
+ nonDictIdx[nonDicIndex++] = allCount;
+ allCount++;
+ }
+
+ // indices for measure columns
+ for (int i = 0; i < measureCount; i++) {
+ measureIdx[i] = allCount;
+ allCount++;
+ }
+ }
+
+ public Object[] convertRow(Object[] data) {
+ // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+ Object[] holder = new Object[3];
+ try {
+
+ int[] dictDims = new int[dimensionCount - noDictionaryCount];
+ byte[][] nonDictArray = new byte[noDictionaryCount + complexDimensionCount][];
+ Object[] measures = new Object[measureCount];
+
+ // write dict dim data
+ for (int idx = 0; idx < dictDimIdx.length; idx++) {
+ dictDims[idx] = (int) data[dictDimIdx[idx]];
+ }
+
+ // write non dict dim data
+ for (int idx = 0; idx < nonDictIdx.length; idx++) {
+ nonDictArray[idx] = (byte[]) data[nonDictIdx[idx]];
+ }
+
+ // write measure data
+ for (int idx = 0; idx < measureIdx.length; idx++) {
+ measures[idx] = data[measureIdx[idx]];
+ }
+ NonDictionaryUtil.prepareOutObj(holder, dictDims, nonDictArray, measures);
+
+ // increment number if record read
+ } catch (Exception e) {
+ throw new RuntimeException("Problem while converting row ", e);
+ }
+ //return out row
+ return holder;
+ }
+}
[43/54] [abbrv] carbondata git commit: [CARBONDATA-2206] support
lucene index datamap
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/bbb10922/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index 80d8154..1c9ccc8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -110,7 +110,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
* @param tablePage
*/
@Override public void writeTablePage(TablePage tablePage)
- throws CarbonDataWriterException {
+ throws CarbonDataWriterException,IOException {
// condition for writting all the pages
if (!tablePage.isLastPage()) {
boolean isAdded = false;
@@ -148,7 +148,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter {
}
}
- private void addPageData(TablePage tablePage) {
+ private void addPageData(TablePage tablePage) throws IOException {
blockletDataHolder.addPage(tablePage);
if (listener != null) {
if (pageId == 0) {
[13/54] [abbrv] carbondata git commit: [CARBONDATA-2099] Refactor
query scan process to improve readability
Posted by ra...@apache.org.
[CARBONDATA-2099] Refactor query scan process to improve readability
Unified concepts in scan process flow:
1.QueryModel contains all parameter for scan, it is created by API in CarbonTable. (In future, CarbonTable will be the entry point for various table operations)
2.Use term ColumnChunk to represent one column in one blocklet, and use ChunkIndex in reader to read specified column chunk
3.Use term ColumnPage to represent one page in one ColumnChunk
4.QueryColumn => ProjectionColumn, indicating it is for projection
This closes #1874
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/daa64650
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/daa64650
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/daa64650
Branch: refs/heads/master
Commit: daa646503ce9ddc5d581b23cbec5494ad8e0be08
Parents: 5ab0957
Author: Jacky Li <ja...@qq.com>
Authored: Tue Jan 30 21:24:04 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:09 2018 +0530
----------------------------------------------------------------------
.../dictionary/AbstractDictionaryCache.java | 3 +-
.../cache/dictionary/DictionaryCacheLoader.java | 7 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 11 +-
.../core/datastore/BTreeBuilderInfo.java | 6 -
.../carbondata/core/datastore/DataRefNode.java | 81 +--
.../carbondata/core/datastore/FileHolder.java | 118 ----
.../carbondata/core/datastore/FileReader.java | 114 +++
.../core/datastore/block/SegmentProperties.java | 50 +-
.../chunk/DimensionColumnDataChunk.java | 116 ---
.../datastore/chunk/DimensionColumnPage.java | 111 +++
.../chunk/impl/AbstractDimensionColumnPage.java | 89 +++
.../chunk/impl/AbstractDimensionDataChunk.java | 95 ---
.../impl/ColumnGroupDimensionColumnPage.java | 194 ++++++
.../impl/ColumnGroupDimensionDataChunk.java | 194 ------
.../chunk/impl/DimensionRawColumnChunk.java | 46 +-
.../impl/FixedLengthDimensionColumnPage.java | 163 +++++
.../impl/FixedLengthDimensionDataChunk.java | 163 -----
.../chunk/impl/MeasureRawColumnChunk.java | 26 +-
.../impl/VariableLengthDimensionColumnPage.java | 133 ++++
.../impl/VariableLengthDimensionDataChunk.java | 140 ----
.../reader/DimensionColumnChunkReader.java | 14 +-
.../chunk/reader/MeasureColumnChunkReader.java | 12 +-
.../AbstractChunkReaderV2V3Format.java | 34 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 38 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 30 +-
...essedDimChunkFileBasedPageLevelReaderV3.java | 11 +-
...mpressedDimensionChunkFileBasedReaderV3.java | 49 +-
.../AbstractMeasureChunkReaderV2V3Format.java | 42 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 16 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 24 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 45 +-
...essedMsrChunkFileBasedPageLevelReaderV3.java | 8 +-
.../chunk/store/ColumnPageWrapper.java | 30 +-
.../chunk/store/DimensionDataChunkStore.java | 8 +-
.../SafeFixedLengthDimensionDataChunkStore.java | 6 +-
...feVariableLengthDimensionDataChunkStore.java | 8 +-
...nsafeFixedLengthDimensionDataChunkStore.java | 10 +-
...afeVariableLengthDimesionDataChunkStore.java | 10 +-
.../datastore/columnar/ColumnGroupModel.java | 26 -
.../core/datastore/impl/DFSFileHolderImpl.java | 166 -----
.../core/datastore/impl/DFSFileReaderImpl.java | 155 ++++
.../datastore/impl/DefaultFileTypeProvider.java | 16 +-
.../core/datastore/impl/FileFactory.java | 4 +-
.../core/datastore/impl/FileHolderImpl.java | 224 ------
.../core/datastore/impl/FileReaderImpl.java | 215 ++++++
.../core/datastore/impl/FileTypeInerface.java | 4 +-
.../impl/btree/AbstractBTreeLeafNode.java | 60 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 6 +-
.../datastore/impl/btree/BTreeNonLeafNode.java | 52 +-
.../impl/btree/BlockBTreeLeafNode.java | 6 +-
.../impl/btree/BlockletBTreeLeafNode.java | 46 +-
.../page/encoding/EncodingFactory.java | 8 +-
.../server/NonSecureDictionaryServer.java | 1 -
.../core/indexstore/BlockletDetailInfo.java | 4 -
.../blockletindex/BlockletDataRefNode.java | 228 ++++++
.../BlockletDataRefNodeWrapper.java | 241 -------
.../indexstore/blockletindex/IndexWrapper.java | 2 +-
.../blockletindex/SegmentIndexFileStore.java | 7 +-
.../core/memory/HeapMemoryAllocator.java | 2 +-
.../core/metadata/blocklet/SegmentInfo.java | 19 -
.../core/metadata/schema/table/CarbonTable.java | 130 +++-
.../schema/table/RelationIdentifier.java | 16 -
.../core/metadata/schema/table/TableInfo.java | 6 +-
.../schema/table/column/CarbonColumn.java | 2 +-
.../schema/table/column/CarbonDimension.java | 12 -
.../core/mutate/CarbonUpdateUtil.java | 20 +-
.../core/mutate/DeleteDeltaBlockDetails.java | 8 -
.../core/mutate/DeleteDeltaBlockletDetails.java | 8 -
.../data/BlockletDeleteDeltaCacheLoader.java | 87 ---
.../scan/collector/ScannedResultCollector.java | 16 +-
.../impl/AbstractScannedResultCollector.java | 29 +-
.../impl/DictionaryBasedResultCollector.java | 53 +-
.../DictionaryBasedVectorResultCollector.java | 63 +-
.../collector/impl/RawBasedResultCollector.java | 32 +-
...structureBasedDictionaryResultCollector.java | 17 +-
.../RestructureBasedRawResultCollector.java | 27 +-
.../RestructureBasedVectorResultCollector.java | 33 +-
.../core/scan/complextypes/ArrayQueryType.java | 4 +-
.../scan/complextypes/ComplexQueryType.java | 12 +-
.../scan/complextypes/PrimitiveQueryType.java | 4 +-
.../core/scan/complextypes/StructQueryType.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 156 ++---
.../scan/executor/impl/DetailQueryExecutor.java | 6 +-
.../scan/executor/infos/BlockExecutionInfo.java | 179 ++---
.../core/scan/executor/util/QueryUtil.java | 141 ++--
.../scan/executor/util/RestructureUtil.java | 70 +-
.../core/scan/expression/ColumnExpression.java | 8 -
.../scan/expression/FilterModificationNode.java | 35 +-
.../expression/RangeExpressionEvaluator.java | 11 +-
.../scan/filter/FilterExpressionProcessor.java | 22 +-
.../core/scan/filter/FilterProcessor.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 60 +-
.../core/scan/filter/GenericQueryType.java | 4 +-
.../filter/executer/AndFilterExecuterImpl.java | 22 +-
.../ExcludeColGroupFilterExecuterImpl.java | 85 ---
.../executer/ExcludeFilterExecuterImpl.java | 131 ++--
.../scan/filter/executer/FilterExecuter.java | 10 +-
.../ImplicitIncludeFilterExecutorImpl.java | 23 +-
.../IncludeColGroupFilterExecuterImpl.java | 31 +-
.../executer/IncludeFilterExecuterImpl.java | 142 ++--
.../filter/executer/OrFilterExecuterImpl.java | 25 +-
.../executer/RangeValueFilterExecuterImpl.java | 224 +++---
.../RestructureExcludeFilterExecutorImpl.java | 21 +-
.../RestructureIncludeFilterExecutorImpl.java | 22 +-
.../executer/RowLevelFilterExecuterImpl.java | 151 ++--
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 121 ++--
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 116 +--
...velRangeLessThanEqualFilterExecuterImpl.java | 113 +--
.../RowLevelRangeLessThanFiterExecuterImpl.java | 122 ++--
.../filter/executer/TrueFilterExecutor.java | 17 +-
.../executer/ValueBasedFilterExecuterImpl.java | 16 +-
.../scan/filter/intf/FilterOptimizerBasic.java | 26 -
.../filter/optimizer/RangeFilterOptmizer.java | 3 +-
.../resolver/ConditionalFilterResolverImpl.java | 18 +-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 2 +-
.../MeasureColumnResolvedFilterInfo.java | 12 -
.../TrueConditionalResolverImpl.java | 16 +-
.../visitor/RangeDictionaryColumnVisitor.java | 2 -
.../core/scan/model/CarbonQueryPlan.java | 151 ----
.../core/scan/model/ProjectionColumn.java | 59 ++
.../core/scan/model/ProjectionDimension.java | 45 ++
.../core/scan/model/ProjectionMeasure.java | 46 ++
.../carbondata/core/scan/model/QueryColumn.java | 66 --
.../core/scan/model/QueryDimension.java | 56 --
.../core/scan/model/QueryMeasure.java | 59 --
.../carbondata/core/scan/model/QueryModel.java | 120 +---
.../core/scan/model/QueryProjection.java | 83 +++
.../processor/AbstractDataBlockIterator.java | 251 -------
.../core/scan/processor/BlockletIterator.java | 14 +-
.../core/scan/processor/BlocksChunkHolder.java | 141 ----
.../core/scan/processor/DataBlockIterator.java | 269 +++++++
.../scan/processor/RawBlockletColumnChunks.java | 113 +++
.../processor/impl/DataBlockIteratorImpl.java | 69 --
.../core/scan/result/AbstractScannedResult.java | 698 -------------------
.../core/scan/result/BatchResult.java | 103 ---
.../core/scan/result/BlockletScannedResult.java | 618 ++++++++++++++++
.../carbondata/core/scan/result/RowBatch.java | 103 +++
.../result/impl/FilterQueryScannedResult.java | 50 +-
.../impl/NonFilterQueryScannedResult.java | 14 +-
.../AbstractDetailQueryResultIterator.java | 44 +-
.../scan/result/iterator/ChunkRowIterator.java | 8 +-
.../iterator/DetailQueryResultIterator.java | 14 +-
.../PartitionSpliterRawResultIterator.java | 15 +-
.../scan/result/iterator/RawResultIterator.java | 10 +-
.../VectorDetailQueryResultIterator.java | 4 +-
.../scan/result/vector/CarbonColumnarBatch.java | 3 -
.../scan/result/vector/ColumnVectorInfo.java | 8 +-
.../vector/MeasureDataVectorProcessor.java | 36 +-
.../scan/scanner/AbstractBlockletScanner.java | 181 -----
.../core/scan/scanner/BlockletScanner.java | 22 +-
.../scanner/impl/BlockletFilterScanner.java | 329 +++++++++
.../scan/scanner/impl/BlockletFullScanner.java | 191 +++++
.../core/scan/scanner/impl/FilterScanner.java | 326 ---------
.../scan/scanner/impl/NonFilterScanner.java | 35 -
.../core/scan/wrappers/ByteArrayWrapper.java | 41 +-
.../core/stats/PartitionStatistic.java | 23 -
.../core/stats/QueryStatisticsModel.java | 3 -
.../core/statusmanager/LoadMetadataDetails.java | 9 -
.../statusmanager/SegmentStatusManager.java | 16 -
.../SegmentUpdateStatusManager.java | 156 +----
.../util/AbstractDataFileFooterConverter.java | 1 -
.../apache/carbondata/core/util/CarbonUtil.java | 34 +-
.../core/util/DataFileFooterConverter.java | 6 +-
.../core/writer/CarbonDictionaryWriter.java | 11 -
.../core/writer/CarbonDictionaryWriterImpl.java | 2 +-
.../DictionaryCacheLoaderImplTest.java | 4 +-
.../store/impl/DFSFileHolderImplUnitTest.java | 149 ----
.../store/impl/DFSFileReaderImplUnitTest.java | 149 ++++
.../store/impl/FileHolderImplUnitTest.java | 154 ----
.../store/impl/FileReaderImplUnitTest.java | 154 ++++
.../datastore/block/SegmentPropertiesTest.java | 8 +-
.../impl/ColumnGroupDimensionDataChunkTest.java | 8 +-
.../impl/FixedLengthDimensionDataChunkTest.java | 8 +-
.../impl/btree/BTreeBlockFinderTest.java | 20 +-
.../impl/RawBasedResultCollectorTest.java | 32 +-
.../core/scan/executor/util/QueryUtilTest.java | 41 +-
.../scan/executor/util/RestructureUtilTest.java | 38 +-
.../core/scan/filter/FilterUtilTest.java | 2 +-
.../executer/ExcludeFilterExecuterImplTest.java | 6 +-
.../executer/IncludeFilterExecuterImplTest.java | 32 +-
.../core/scan/result/BatchResultTest.java | 94 ---
.../core/scan/result/RowBatchTest.java | 94 +++
.../carbondata/core/util/CarbonUtilTest.java | 90 +--
.../core/util/DataFileFooterConverterTest.java | 11 +-
.../core/util/RangeFilterProcessorTest.java | 9 +-
.../scanner/impl/FilterScannerTest.java | 160 -----
dev/findbugs-exclude.xml | 2 +-
.../examples/CarbonSessionExample.scala | 2 +-
.../hadoop/api/CarbonTableInputFormat.java | 44 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../streaming/CarbonStreamRecordReader.java | 2 +-
.../hadoop/util/BlockLevelTraverser.java | 2 +-
.../hadoop/util/CarbonInputFormatUtil.java | 73 +-
.../hive/MapredCarbonInputFormat.java | 9 +-
.../presto/CarbonVectorizedRecordReader.java | 243 -------
.../carbondata/presto/CarbondataPageSource.java | 2 +-
.../presto/CarbondataRecordCursor.java | 6 +-
.../carbondata/presto/CarbondataRecordSet.java | 4 +-
.../presto/CarbondataRecordSetProvider.java | 2 +-
.../PrestoCarbonVectorizedRecordReader.java | 243 +++++++
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 3 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../vectorreader/ColumnarVectorWrapper.java | 2 +-
.../VectorizedCarbonRecordReader.java | 51 +-
.../merger/CarbonCompactionExecutor.java | 64 +-
.../partition/impl/QueryPartitionHelper.java | 6 +-
.../spliter/AbstractCarbonQueryExecutor.java | 54 +-
.../partition/spliter/CarbonSplitExecutor.java | 4 +-
.../processing/util/CarbonQueryUtil.java | 6 +-
.../carbondata/streaming/StreamHandoffRDD.scala | 2 +-
211 files changed, 5846 insertions(+), 7402 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index 598d00e..fb67208 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -260,8 +260,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
DictionaryCacheLoader dictionaryCacheLoader =
new DictionaryCacheLoaderImpl(dictionaryColumnUniqueIdentifier);
dictionaryCacheLoader
- .load(dictionaryInfo, dictionaryColumnUniqueIdentifier.getColumnIdentifier(),
- dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
+ .load(dictionaryInfo, dictionaryChunkStartOffset, dictionaryChunkEndOffset, loadSortIndex);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
index 01c277d..8df1539 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
@@ -19,8 +19,6 @@ package org.apache.carbondata.core.cache.dictionary;
import java.io.IOException;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-
public interface DictionaryCacheLoader {
/**
@@ -28,7 +26,6 @@ public interface DictionaryCacheLoader {
*
* @param dictionaryInfo dictionary info object which will hold the required data
* for a given column
- * @param columnIdentifier column unique identifier
* @param dictionaryChunkStartOffset start offset from where dictionary file has to
* be read
* @param dictionaryChunkEndOffset end offset till where dictionary file has to
@@ -37,7 +34,7 @@ public interface DictionaryCacheLoader {
* read in memory after dictionary loading
* @throws IOException
*/
- void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
- long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+ void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
+ long dictionaryChunkEndOffset, boolean loadSortIndex)
throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index a603c01..899abf5 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -23,7 +23,6 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.apache.carbondata.core.service.CarbonCommonFactory;
@@ -43,8 +42,7 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
/**
* @param dictionaryColumnUniqueIdentifier dictionary column identifier
*/
- public DictionaryCacheLoaderImpl(
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ DictionaryCacheLoaderImpl(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
this.dictionaryColumnUniqueIdentifier = dictionaryColumnUniqueIdentifier;
}
@@ -53,7 +51,6 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
*
* @param dictionaryInfo dictionary info object which will hold the required data
* for a given column
- * @param columnIdentifier column unique identifier
* @param dictionaryChunkStartOffset start offset from where dictionary file has to
* be read
* @param dictionaryChunkEndOffset end offset till where dictionary file has to
@@ -62,9 +59,9 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
* read in memory after dictionary loading
* @throws IOException
*/
- @Override public void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
- long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
- throws IOException {
+ @Override
+ public void load(DictionaryInfo dictionaryInfo, long dictionaryChunkStartOffset,
+ long dictionaryChunkEndOffset, boolean loadSortIndex) throws IOException {
Iterator<byte[]> columnDictionaryChunkWrapper =
load(dictionaryColumnUniqueIdentifier, dictionaryChunkStartOffset,
dictionaryChunkEndOffset);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
index 3bfbc45..381e764 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BTreeBuilderInfo.java
@@ -43,16 +43,10 @@ public class BTreeBuilderInfo {
this.footerList = footerList;
}
- /**
- * @return the eachDimensionBlockSize
- */
public int[] getDimensionColumnValueSize() {
return dimensionColumnValueSize;
}
- /**
- * @return the footerList
- */
public List<DataFileFooter> getFooterList() {
return footerList;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 13d5f69..273f833 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -18,7 +18,6 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -28,46 +27,47 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
public interface DataRefNode {
/**
- * Method to get the next block this can be used while scanning when
+ * Return the next data block in the tree, this can be used while scanning when
* iterator of this class can be used iterate over blocks
- *
- * @return next block
*/
DataRefNode getNextDataRefNode();
/**
- * to get the number of keys tuples present in the block
- *
- * @return number of keys in the block
+ * Return the number of rows in the data block
*/
- int nodeSize();
+ int numRows();
/**
- * Method can be used to get the block index .This can be used when multiple
- * thread can be used scan group of blocks in that can we can assign the
+ * Return the block index. This can be used when multiple
+ * thread can be used scan group of blocks in that can we can assign
* some of the blocks to one thread and some to other
- *
- * @return block number
*/
- long nodeNumber();
+ long nodeIndex();
+
+ /**
+ * Return the blocklet index in the node
+ */
+ short blockletIndex();
/**
- * Method is used for retreiving the BlockletId.
- * @return the blockletid related to the data block.
+ * Return the number of pages
*/
- String blockletId();
+ int numberOfPages();
/**
- * This method will be used to get the max value of all the columns this can
+ * Return the number of rows for a give page
+ */
+ int getPageRowCount(int pageNumber);
+
+ /**
+ * Return the max value of all the columns, this can
* be used in case of filter query
- *
*/
byte[][] getColumnsMaxValue();
/**
- * This method will be used to get the min value of all the columns this can
+ * Return the min value of all the columns, this can
* be used in case of filter query
- *
*/
byte[][] getColumnsMinValue();
@@ -75,15 +75,15 @@ public interface DataRefNode {
* Below method will be used to get the dimension chunks
*
* @param fileReader file reader to read the chunks from file
- * @param blockIndexes range indexes of the blocks need to be read
+ * @param columnIndexRange range indexes of the blocks need to be read
* value can be {{0,10},{11,12},{13,13}}
* here 0 to 10 and 11 to 12 column blocks will be read in one
* IO operation 13th column block will be read separately
* This will be helpful to reduce IO by reading bigger chunk of
- * data in On IO
+ * data in one IO operation
* @return dimension data chunks
*/
- DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
+ DimensionRawColumnChunk[] readDimensionChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
@@ -92,54 +92,31 @@ public interface DataRefNode {
* @param fileReader file reader to read the chunk from file
* @return dimension data chunk
*/
- DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
+ DimensionRawColumnChunk readDimensionChunk(FileReader fileReader, int columnIndex)
throws IOException;
/**
* Below method will be used to get the measure chunk
*
* @param fileReader file reader to read the chunk from file
- * @param blockIndexes range indexes of the blocks need to be read
+ * @param columnIndexRange range indexes of the blocks need to be read
* value can be {{0,10},{11,12},{13,13}}
* here 0 to 10 and 11 to 12 column blocks will be read in one
* IO operation 13th column block will be read separately
* This will be helpful to reduce IO by reading bigger chunk of
- * data in On IO
+ * data in one IO operation
* @return measure column data chunk
*/
- MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ MeasureRawColumnChunk[] readMeasureChunks(FileReader fileReader, int[][] columnIndexRange)
throws IOException;
/**
* Below method will be used to read the measure chunk
*
* @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
+ * @param columnIndex block index to be read from file
* @return measure data chunk
*/
- MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex) throws IOException;
+ MeasureRawColumnChunk readMeasureChunk(FileReader fileReader, int columnIndex) throws IOException;
- /**
- * @param deleteDeltaDataCache
- */
- void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache);
-
- /**
- * @return
- */
- BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache();
-
- /**
- * number of pages in blocklet
- * @return
- */
- int numberOfPages();
-
- /**
- * Return the number of rows for a give page
- *
- * @param pageNumber
- * @return
- */
- int getPageRowCount(int pageNumber);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
deleted file mode 100644
index b4130a0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-public interface FileHolder {
-
- /**
- * This method will be used to reads the data to byteBuffer from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return ByteBuffer
- * @throws IOException
- */
- ByteBuffer readByteBuffer(String filePath, long offset, int length)
- throws IOException;
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, long offset, int length) throws IOException;
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, int length) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- int readInt(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readLong(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- int readInt(String filePath) throws IOException;
-
- /**
- * This method will be used to read long value from file from postion(offset), here
- * length will be always 8 because long byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readDouble(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- void finish() throws IOException;
-
- void setQueryId(String queryId);
-
- String getQueryId();
-
- /**
- * Set the flag to read data page by page instead of whole blocklet.
- *
- * @param isReadPageByPage
- */
- void setReadPageByPage(boolean isReadPageByPage);
-
- /**
- * Whether read the data page by page from carbondata file instead of reading whole
- * blocklet to memory. It is need in case of memory constraint operations.
- */
- boolean isReadPageByPage();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
new file mode 100644
index 0000000..df0d745
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileReader.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface FileReader {
+
+ /**
+ * This method will be used to reads the data to byteBuffer from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return ByteBuffer
+ * @throws IOException
+ */
+ ByteBuffer readByteBuffer(String filePath, long offset, int length)
+ throws IOException;
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath) throws IOException;
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish() throws IOException;
+
+ /**
+ * Set the flag to read data page by page instead of whole blocklet.
+ *
+ * @param isReadPageByPage
+ */
+ void setReadPageByPage(boolean isReadPageByPage);
+
+ /**
+ * Whether read the data page by page from carbondata file instead of reading whole
+ * blocklet to memory. It is need in case of memory constraint operations.
+ */
+ boolean isReadPageByPage();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 5f82e87..c134db9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -94,10 +94,9 @@ public class SegmentProperties {
private int[] complexDimColumnCardinality;
/**
- * mapping of dimension column to block in a file this will be used for
- * reading the blocks from file
+ * mapping of dimension ordinal in schema to column chunk index in the data file
*/
- private Map<Integer, Integer> dimensionOrdinalToBlockMapping;
+ private Map<Integer, Integer> dimensionOrdinalToChunkMapping;
/**
* a block can have multiple columns. This will have block index as key
@@ -106,10 +105,9 @@ public class SegmentProperties {
private Map<Integer, Set<Integer>> blockTodimensionOrdinalMapping;
/**
- * mapping of measure column to block to in file this will be used while
- * reading the block in a file
+ * mapping of measure ordinal in schema to column chunk index in the data file
*/
- private Map<Integer, Integer> measuresOrdinalToBlockMapping;
+ private Map<Integer, Integer> measuresOrdinalToChunkMapping;
/**
* size of the each dimension column value in a block this can be used when
@@ -172,15 +170,15 @@ public class SegmentProperties {
new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
measures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
fillDimensionAndMeasureDetails(columnsInTable, columnCardinality);
- dimensionOrdinalToBlockMapping =
+ dimensionOrdinalToChunkMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
blockTodimensionOrdinalMapping =
new HashMap<Integer, Set<Integer>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- measuresOrdinalToBlockMapping =
+ measuresOrdinalToChunkMapping =
new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
intialiseColGroups();
fillOrdinalToBlockMappingForDimension();
- fillOrdinalToBlockIndexMappingForMeasureColumns();
+ fillOrdinalToChunkIndexMappingForMeasureColumns();
fillColumnGroupAndItsCardinality(columnCardinality);
fillKeyGeneratorDetails();
}
@@ -237,7 +235,7 @@ public class SegmentProperties {
if (dimension.isColumnar() || dimension.columnGroupId() != prvcolumnGroupId) {
blockOrdinal++;
}
- dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), blockOrdinal);
+ dimensionOrdinalToChunkMapping.put(dimension.getOrdinal(), blockOrdinal);
prvcolumnGroupId = dimension.columnGroupId();
index++;
}
@@ -245,7 +243,7 @@ public class SegmentProperties {
// complex dimension will be stored at last
while (index < complexDimensions.size()) {
dimension = complexDimensions.get(index);
- dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), ++blockOrdinal);
+ dimensionOrdinalToChunkMapping.put(dimension.getOrdinal(), ++blockOrdinal);
blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal, dimension);
index++;
}
@@ -256,7 +254,7 @@ public class SegmentProperties {
*
*/
private void fillBlockToDimensionOrdinalMapping() {
- Set<Entry<Integer, Integer>> blocks = dimensionOrdinalToBlockMapping.entrySet();
+ Set<Entry<Integer, Integer>> blocks = dimensionOrdinalToChunkMapping.entrySet();
Iterator<Entry<Integer, Integer>> blockItr = blocks.iterator();
while (blockItr.hasNext()) {
Entry<Integer, Integer> block = blockItr.next();
@@ -280,7 +278,7 @@ public class SegmentProperties {
*/
private int fillComplexDimensionChildBlockIndex(int blockOrdinal, CarbonDimension dimension) {
for (int i = 0; i < dimension.getNumberOfChild(); i++) {
- dimensionOrdinalToBlockMapping
+ dimensionOrdinalToChunkMapping
.put(dimension.getListOfChildDimensions().get(i).getOrdinal(), ++blockOrdinal);
if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal,
@@ -295,11 +293,11 @@ public class SegmentProperties {
* of measure ordinal to its block index mapping in
* file
*/
- private void fillOrdinalToBlockIndexMappingForMeasureColumns() {
+ private void fillOrdinalToChunkIndexMappingForMeasureColumns() {
int blockOrdinal = 0;
int index = 0;
while (index < measures.size()) {
- measuresOrdinalToBlockMapping.put(measures.get(index).getOrdinal(), blockOrdinal);
+ measuresOrdinalToChunkMapping.put(measures.get(index).getOrdinal(), blockOrdinal);
blockOrdinal++;
index++;
}
@@ -731,17 +729,17 @@ public class SegmentProperties {
}
/**
- * @return the dimensionOrdinalToBlockMapping
+ * @return the dimensionOrdinalToChunkMapping
*/
- public Map<Integer, Integer> getDimensionOrdinalToBlockMapping() {
- return dimensionOrdinalToBlockMapping;
+ public Map<Integer, Integer> getDimensionOrdinalToChunkMapping() {
+ return dimensionOrdinalToChunkMapping;
}
/**
- * @return the measuresOrdinalToBlockMapping
+ * @return the measuresOrdinalToChunkMapping
*/
- public Map<Integer, Integer> getMeasuresOrdinalToBlockMapping() {
- return measuresOrdinalToBlockMapping;
+ public Map<Integer, Integer> getMeasuresOrdinalToChunkMapping() {
+ return measuresOrdinalToChunkMapping;
}
/**
@@ -805,16 +803,6 @@ public class SegmentProperties {
}
/**
- * It returns no of column availble in given column group
- *
- * @param colGrpId
- * @return no of column in given column group
- */
- public int getNoOfColumnsInColumnGroup(int colGrpId) {
- return columnGroupOrdinalToMdkeymapping.get(colGrpId).size();
- }
-
- /**
* @param blockIndex
* @return It returns all dimension present in given block index
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
deleted file mode 100644
index 3791314..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk;
-
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * Interface for dimension column chunk.
- */
-public interface DimensionColumnDataChunk {
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @return how many bytes was copied
- */
- int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
-
- /**
- * It uses to convert column data to dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param restructuringInfo @return
- */
- int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Fill the data to vector
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Fill the data to vector
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo);
-
- /**
- * Below method to get the data based in row id
- *
- * @return chunk
- */
- byte[] getChunkData(int columnIndex);
-
- /**
- * @return inverted index
- */
- int getInvertedIndex(int index);
-
- /**
- *
- * @param invertedIndex
- * @return index reverse index
- */
- int getInvertedReverseIndex(int invertedIndex);
-
- /**
- * @return whether column is dictionary column or not
- */
- boolean isNoDicitionaryColumn();
-
- /**
- * @return length of each column
- */
- int getColumnValueSize();
-
- /**
- * @return whether columns where explictly sorted or not
- */
- boolean isExplicitSorted();
-
- /**
- * to compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- int compareTo(int index, byte[] compareValue);
-
- /**
- * below method will be used to free the allocated memory
- */
- void freeMemory();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
new file mode 100644
index 0000000..15840bc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnPage.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk;
+
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * Interface for dimension column chunk.
+ */
+public interface DimensionColumnPage {
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @return how many bytes was copied
+ */
+ int fillRawData(int rowId, int offset, byte[] data, KeyStructureInfo restructuringInfo);
+
+ /**
+ * It uses to convert column data to dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo @return
+ */
+ int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Fill the data to vector
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Fill the data to vector
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @return chunk
+ */
+ byte[] getChunkData(int rowId);
+
+ /**
+ * @return inverted index
+ */
+ int getInvertedIndex(int rowId);
+
+ /**
+ *
+ * @param rowId
+ * @return index reverse index
+ */
+ int getInvertedReverseIndex(int rowId);
+
+ /**
+ * @return whether column is dictionary column or not
+ */
+ boolean isNoDicitionaryColumn();
+
+ /**
+ * @return whether columns where explictly sorted or not
+ */
+ boolean isExplicitSorted();
+
+ /**
+ * to compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ int compareTo(int rowId, byte[] compareValue);
+
+ /**
+ * below method will be used to free the allocated memory
+ */
+ void freeMemory();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
new file mode 100644
index 0000000..6f316c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionColumnPage.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
+
+/**
+ * Class responsibility is to give access to dimension column data chunk store
+ */
+public abstract class AbstractDimensionColumnPage implements DimensionColumnPage {
+
+ /**
+ * data chunks
+ */
+ DimensionDataChunkStore dataChunkStore;
+
+ /**
+ * @return whether columns where explicitly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return dataChunkStore.isExplicitSorted();
+ }
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param rowId row id of the data
+ * @return chunk
+ */
+ @Override public byte[] getChunkData(int rowId) {
+ return dataChunkStore.getRow(rowId);
+ }
+
+ /**
+ * @return inverted index
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ return dataChunkStore.getInvertedIndex(rowId);
+ }
+
+ /**
+ * @param rowId
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ return dataChunkStore.getInvertedReverseIndex(rowId);
+ }
+
+ /**
+ * To compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ @Override public int compareTo(int rowId, byte[] compareValue) {
+ // TODO Auto-generated method stub
+ return dataChunkStore.compareTo(rowId, compareValue);
+ }
+
+ /**
+ * below method will be used to free the allocated memory
+ */
+ @Override public void freeMemory() {
+ dataChunkStore.freeMemory();
+ }
+
+ /**
+ * @return column is dictionary column or not
+ */
+ @Override public boolean isNoDicitionaryColumn() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
deleted file mode 100644
index eac062f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
-
-/**
- * Class responsibility is to give access to dimension column data chunk store
- */
-public abstract class AbstractDimensionDataChunk implements DimensionColumnDataChunk {
-
- /**
- * data chunks
- */
- protected DimensionDataChunkStore dataChunkStore;
-
- /**
- * @return whether columns where explicitly sorted or not
- */
- @Override public boolean isExplicitSorted() {
- return dataChunkStore.isExplicitSorted();
- }
-
- /**
- * Below method to get the data based in row id
- *
- * @param index row id of the data
- * @return chunk
- */
- @Override public byte[] getChunkData(int index) {
- return dataChunkStore.getRow(index);
- }
-
- /**
- * @return inverted index
- */
- @Override public int getInvertedIndex(int index) {
- return dataChunkStore.getInvertedIndex(index);
- }
-
- /**
- * @param invertedIndex
- * @return inverted index reverse
- */
- @Override public int getInvertedReverseIndex(int invertedIndex) {
- return dataChunkStore.getInvertedReverseIndex(invertedIndex);
- }
- /**
- * @return length of each column
- */
- @Override public int getColumnValueSize() {
- return dataChunkStore.getColumnValueSize();
- }
-
- /**
- * To compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- @Override public int compareTo(int index, byte[] compareValue) {
- // TODO Auto-generated method stub
- return dataChunkStore.compareTo(index, compareValue);
- }
-
- /**
- * below method will be used to free the allocated memory
- */
- @Override public void freeMemory() {
- dataChunkStore.freeMemory();
- }
-
- /**
- * @return column is dictionary column or not
- */
- @Override public boolean isNoDicitionaryColumn() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
new file mode 100644
index 0000000..741c13d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionColumnPage.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to column group dimension data chunk store
+ */
+public class ColumnGroupDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor for this class
+ *
+ * @param dataChunk data chunk
+ * @param columnValueSize chunk attributes
+ * @param numberOfRows
+ */
+ public ColumnGroupDimensionColumnPage(byte[] dataChunk, int columnValueSize, int numberOfRows) {
+ this.dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(columnValueSize, false, numberOfRows, dataChunk.length,
+ DimensionStoreType.FIXEDLENGTH);
+ this.dataChunkStore.putArray(null, null, dataChunk);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo restructuringInfo) {
+ byte[] row = dataChunkStore.getRow(rowId);
+ byte[] maskedKey = getMaskedKey(row, restructuringInfo);
+ System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
+ return maskedKey.length;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param info KeyStructureInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo info) {
+ byte[] data = dataChunkStore.getRow(rowId);
+ long[] keyArray = info.getKeyGenerator().getKeyArray(data);
+ int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+ for (int i = 0; i < ordinal.length; i++) {
+ outputSurrogateKey[chunkIndex++] = (int) keyArray[ordinal[i]];
+ }
+ return chunkIndex;
+ }
+
+ /**
+ * Below method will be used to get the masked key
+ *
+ * @param data data
+ * @param info
+ * @return
+ */
+ private byte[] getMaskedKey(byte[] data, KeyStructureInfo info) {
+ byte[] maskedKey = new byte[info.getMaskByteRanges().length];
+ int counter = 0;
+ int byteRange = 0;
+ for (int i = 0; i < info.getMaskByteRanges().length; i++) {
+ byteRange = info.getMaskByteRanges()[i];
+ maskedKey[counter++] = (byte) (data[byteRange] & info.getMaxKey()[byteRange]);
+ }
+ return maskedKey;
+ }
+
+ /**
+ * @return inverted index
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * @param rowId
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * @return whether columns where explictly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return false;
+ }
+
+ /**
+ * to compare the data
+ *
+ * @param rowId row index to be compared
+ * @param compareValue value to compare
+ * @return compare result
+ */
+ @Override public int compareTo(int rowId, byte[] compareValue) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
+ for (int k = offset; k < len; k++) {
+ long[] keyArray = restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(k));
+ int index = 0;
+ for (int i = chunkIndex; i < chunkIndex + ordinal.length; i++) {
+ if (vectorInfo[i].directDictionaryGenerator == null) {
+ vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
+ } else {
+ vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
+ .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
+ }
+ }
+ vectorOffset++;
+ }
+ return chunkIndex + ordinal.length;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = offset + columnVectorInfo.size;
+ int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
+ for (int k = offset; k < len; k++) {
+ long[] keyArray =
+ restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(filteredRowId[k]));
+ int index = 0;
+ for (int i = chunkIndex; i < chunkIndex + ordinal.length; i++) {
+ if (vectorInfo[i].directDictionaryGenerator == null) {
+ vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
+ } else {
+ vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
+ .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
+ }
+ }
+ vectorOffset++;
+ }
+ return chunkIndex + ordinal.length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
deleted file mode 100644
index b76ae53..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
-
-/**
- * This class is gives access to column group dimension data chunk store
- */
-public class ColumnGroupDimensionDataChunk extends AbstractDimensionDataChunk {
-
- /**
- * Constructor for this class
- *
- * @param dataChunk data chunk
- * @param columnValueSize chunk attributes
- * @param numberOfRows
- */
- public ColumnGroupDimensionDataChunk(byte[] dataChunk, int columnValueSize, int numberOfRows) {
- this.dataChunkStore = DimensionChunkStoreFactory.INSTANCE
- .getDimensionChunkStore(columnValueSize, false, numberOfRows, dataChunk.length,
- DimensionStoreType.FIXEDLENGTH);
- this.dataChunkStore.putArray(null, null, dataChunk);
- }
-
- /**
- * Below method will be used to fill the data based on offset and row id
- *
- * @param data data to filed
- * @param offset offset from which data need to be filed
- * @param rowId row id of the chunk
- * @param restructuringInfo define the structure of the key
- * @return how many bytes was copied
- */
- @Override public int fillChunkData(byte[] data, int offset, int rowId,
- KeyStructureInfo restructuringInfo) {
- byte[] row = dataChunkStore.getRow(rowId);
- byte[] maskedKey = getMaskedKey(row, restructuringInfo);
- System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
- return maskedKey.length;
- }
-
- /**
- * Converts to column dictionary integer value
- *
- * @param rowId
- * @param columnIndex
- * @param row
- * @param info KeyStructureInfo
- * @return
- */
- @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
- KeyStructureInfo info) {
- byte[] data = dataChunkStore.getRow(rowId);
- long[] keyArray = info.getKeyGenerator().getKeyArray(data);
- int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
- for (int i = 0; i < ordinal.length; i++) {
- row[columnIndex++] = (int) keyArray[ordinal[i]];
- }
- return columnIndex;
- }
-
- /**
- * Below method will be used to get the masked key
- *
- * @param data data
- * @param info
- * @return
- */
- private byte[] getMaskedKey(byte[] data, KeyStructureInfo info) {
- byte[] maskedKey = new byte[info.getMaskByteRanges().length];
- int counter = 0;
- int byteRange = 0;
- for (int i = 0; i < info.getMaskByteRanges().length; i++) {
- byteRange = info.getMaskByteRanges()[i];
- maskedKey[counter++] = (byte) (data[byteRange] & info.getMaxKey()[byteRange]);
- }
- return maskedKey;
- }
-
- /**
- * @return inverted index
- */
- @Override public int getInvertedIndex(int index) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * @param invertedIndex
- * @return inverted index reverse
- */
- @Override public int getInvertedReverseIndex(int invertedIndex) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * @return whether columns where explictly sorted or not
- */
- @Override public boolean isExplicitSorted() {
- return false;
- }
-
- /**
- * to compare the data
- *
- * @param index row index to be compared
- * @param compareValue value to compare
- * @return compare result
- */
- @Override public int compareTo(int index, byte[] compareValue) {
- throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
- }
-
- /**
- * Fill the data to vector
- *
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
- KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
- for (int k = offset; k < len; k++) {
- long[] keyArray = restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(k));
- int index = 0;
- for (int i = column; i < column + ordinal.length; i++) {
- if (vectorInfo[i].directDictionaryGenerator == null) {
- vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
- } else {
- vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
- .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
- }
- }
- vectorOffset++;
- }
- return column + ordinal.length;
- }
-
- /**
- * Fill the data to vector
- *
- * @param rowMapping
- * @param vectorInfo
- * @param column
- * @param restructuringInfo
- * @return next column index
- */
- @Override public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
- int column, KeyStructureInfo restructuringInfo) {
- ColumnVectorInfo columnVectorInfo = vectorInfo[column];
- int offset = columnVectorInfo.offset;
- int vectorOffset = columnVectorInfo.vectorOffset;
- int len = offset + columnVectorInfo.size;
- int[] ordinal = restructuringInfo.getMdkeyQueryDimensionOrdinal();
- for (int k = offset; k < len; k++) {
- long[] keyArray =
- restructuringInfo.getKeyGenerator().getKeyArray(dataChunkStore.getRow(rowMapping[k]));
- int index = 0;
- for (int i = column; i < column + ordinal.length; i++) {
- if (vectorInfo[i].directDictionaryGenerator == null) {
- vectorInfo[i].vector.putInt(vectorOffset, (int) keyArray[ordinal[index++]]);
- } else {
- vectorInfo[i].vector.putLong(vectorOffset, (long) vectorInfo[i].directDictionaryGenerator
- .getValueFromSurrogate((int) keyArray[ordinal[index++]]));
- }
- }
- vectorOffset++;
- }
- return column + ordinal.length;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
index 705c13c..f9bb590 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/DimensionRawColumnChunk.java
@@ -19,25 +19,25 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.FileReader;
import org.apache.carbondata.core.datastore.chunk.AbstractRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.memory.MemoryException;
/**
* Contains raw dimension data,
* 1. The read uncompressed raw data of column chunk with all pages is stored in this instance.
- * 2. The raw data can be converted to processed chunk using convertToDimColDataChunk method
+ * 2. The raw data can be converted to processed chunk using decodeColumnPage method
* by specifying page number.
*/
public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
- private DimensionColumnDataChunk[] dataChunks;
+ private DimensionColumnPage[] dataChunks;
private DimensionColumnChunkReader chunkReader;
- private FileHolder fileHolder;
+ private FileReader fileReader;
public DimensionRawColumnChunk(int columnIndex, ByteBuffer rawData, long offSet, int length,
DimensionColumnChunkReader columnChunkReader) {
@@ -46,17 +46,17 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
/**
- * Convert all raw data with all pages to processed DimensionColumnDataChunk's
+ * Convert all raw data with all pages to processed DimensionColumnPage's
* @return
*/
- public DimensionColumnDataChunk[] convertToDimColDataChunks() {
+ public DimensionColumnPage[] decodeAllColumnPages() {
if (dataChunks == null) {
- dataChunks = new DimensionColumnDataChunk[pagesCount];
+ dataChunks = new DimensionColumnPage[pagesCount];
}
for (int i = 0; i < pagesCount; i++) {
try {
if (dataChunks[i] == null) {
- dataChunks[i] = chunkReader.convertToDimensionChunk(this, i);
+ dataChunks[i] = chunkReader.decodeColumnPage(this, i);
}
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
@@ -66,24 +66,24 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
/**
- * Convert raw data with specified page number processed to DimensionColumnDataChunk
- * @param index
+ * Convert raw data with specified page number processed to DimensionColumnPage
+ * @param pageNumber
* @return
*/
- public DimensionColumnDataChunk convertToDimColDataChunk(int index) {
- assert index < pagesCount;
+ public DimensionColumnPage decodeColumnPage(int pageNumber) {
+ assert pageNumber < pagesCount;
if (dataChunks == null) {
- dataChunks = new DimensionColumnDataChunk[pagesCount];
+ dataChunks = new DimensionColumnPage[pagesCount];
}
- if (dataChunks[index] == null) {
+ if (dataChunks[pageNumber] == null) {
try {
- dataChunks[index] = chunkReader.convertToDimensionChunk(this, index);
+ dataChunks[pageNumber] = chunkReader.decodeColumnPage(this, pageNumber);
} catch (IOException | MemoryException e) {
throw new RuntimeException(e);
}
}
- return dataChunks[index];
+ return dataChunks[pageNumber];
}
/**
@@ -92,10 +92,10 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
* @param index
* @return
*/
- public DimensionColumnDataChunk convertToDimColDataChunkWithOutCache(int index) {
+ public DimensionColumnPage convertToDimColDataChunkWithOutCache(int index) {
assert index < pagesCount;
try {
- return chunkReader.convertToDimensionChunk(this, index);
+ return chunkReader.decodeColumnPage(this, index);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -111,11 +111,11 @@ public class DimensionRawColumnChunk extends AbstractRawColumnChunk {
}
}
- public void setFileHolder(FileHolder fileHolder) {
- this.fileHolder = fileHolder;
+ public void setFileReader(FileReader fileReader) {
+ this.fileReader = fileReader;
}
- public FileHolder getFileReader() {
- return fileHolder;
+ public FileReader getFileReader() {
+ return fileReader;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/daa64650/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
new file mode 100644
index 0000000..ff54b12
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionColumnPage.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.impl;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * This class is gives access to fixed length dimension data chunk store
+ */
+public class FixedLengthDimensionColumnPage extends AbstractDimensionColumnPage {
+
+ /**
+ * Constructor
+ *
+ * @param dataChunk data chunk
+ * @param invertedIndex inverted index
+ * @param invertedIndexReverse reverse inverted index
+ * @param numberOfRows number of rows
+ * @param columnValueSize size of each column value
+ */
+ public FixedLengthDimensionColumnPage(byte[] dataChunk, int[] invertedIndex,
+ int[] invertedIndexReverse, int numberOfRows, int columnValueSize) {
+ long totalSize = null != invertedIndex ?
+ dataChunk.length + (2 * numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE) :
+ dataChunk.length;
+ dataChunkStore = DimensionChunkStoreFactory.INSTANCE
+ .getDimensionChunkStore(columnValueSize, null != invertedIndex, numberOfRows, totalSize,
+ DimensionStoreType.FIXEDLENGTH);
+ dataChunkStore.putArray(invertedIndex, invertedIndexReverse, dataChunk);
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param rowId row id of the chunk
+ * @param offset offset from which data need to be filed
+ * @param data data to filed
+ * @param keyStructureInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillRawData(int rowId, int offset, byte[] data,
+ KeyStructureInfo keyStructureInfo) {
+ dataChunkStore.fillRow(rowId, data, offset);
+ return dataChunkStore.getColumnValueSize();
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ *
+ * @param rowId
+ * @param chunkIndex
+ * @param outputSurrogateKey
+ * @param restructuringInfo
+ * @return
+ */
+ @Override public int fillSurrogateKey(int rowId, int chunkIndex, int[] outputSurrogateKey,
+ KeyStructureInfo restructuringInfo) {
+ outputSurrogateKey[chunkIndex] = dataChunkStore.getSurrogate(rowId);
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(ColumnVectorInfo[] vectorInfo, int chunkIndex,
+ KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = columnVectorInfo.size + offset;
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ for (int j = offset; j < len; j++) {
+ int dict = dataChunkStore.getSurrogate(j);
+ if (columnVectorInfo.directDictionaryGenerator == null) {
+ vector.putInt(vectorOffset++, dict);
+ } else {
+ Object valueFromSurrogate =
+ columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
+ if (valueFromSurrogate == null) {
+ vector.putNull(vectorOffset++);
+ } else {
+ DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
+ if (dataType == DataTypes.INT) {
+ vector.putInt(vectorOffset++, (int) valueFromSurrogate);
+ } else if (dataType == DataTypes.LONG) {
+ vector.putLong(vectorOffset++, (long) valueFromSurrogate);
+ } else {
+ throw new IllegalArgumentException("unsupported data type: " +
+ columnVectorInfo.directDictionaryGenerator.getReturnType());
+ }
+ }
+ }
+ }
+ return chunkIndex + 1;
+ }
+
+ /**
+ * Fill the data to vector
+ *
+ * @param filteredRowId
+ * @param vectorInfo
+ * @param chunkIndex
+ * @param restructuringInfo
+ * @return next column index
+ */
+ @Override public int fillVector(int[] filteredRowId, ColumnVectorInfo[] vectorInfo,
+ int chunkIndex, KeyStructureInfo restructuringInfo) {
+ ColumnVectorInfo columnVectorInfo = vectorInfo[chunkIndex];
+ int offset = columnVectorInfo.offset;
+ int vectorOffset = columnVectorInfo.vectorOffset;
+ int len = columnVectorInfo.size + offset;
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ for (int j = offset; j < len; j++) {
+ int dict = dataChunkStore.getSurrogate(filteredRowId[j]);
+ if (columnVectorInfo.directDictionaryGenerator == null) {
+ vector.putInt(vectorOffset++, dict);
+ } else {
+ Object valueFromSurrogate =
+ columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
+ if (valueFromSurrogate == null) {
+ vector.putNull(vectorOffset++);
+ } else {
+ DataType dataType = columnVectorInfo.directDictionaryGenerator.getReturnType();
+ if (dataType == DataTypes.INT) {
+ vector.putInt(vectorOffset++, (int) valueFromSurrogate);
+ } else if (dataType == DataTypes.LONG) {
+ vector.putLong(vectorOffset++, (long) valueFromSurrogate);
+ } else {
+ throw new IllegalArgumentException("unsupported data type: " +
+ columnVectorInfo.directDictionaryGenerator.getReturnType());
+ }
+ }
+ }
+ }
+ return chunkIndex + 1;
+ }
+}
[36/54] [abbrv] carbondata git commit: [HOTFIX] Fix timestamp issue
in TestSortColumnsWithUnsafe
Posted by ra...@apache.org.
[HOTFIX] Fix timestamp issue in TestSortColumnsWithUnsafe
Fix timestamp issue in TestSortColumnsWithUnsafe
This closes #2001
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a80db8e8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a80db8e8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a80db8e8
Branch: refs/heads/master
Commit: a80db8e851a6e0b7061a013b5a9149e4f7041984
Parents: fc2a7eb
Author: Jacky Li <ja...@qq.com>
Authored: Tue Feb 27 13:06:02 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:11 2018 +0530
----------------------------------------------------------------------
.../testsuite/sortcolumns/TestSortColumns.scala | 14 +++++------
.../sortcolumns/TestSortColumnsWithUnsafe.scala | 25 +++++++-------------
2 files changed, 15 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a80db8e8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 13db652..adf8423 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -37,7 +37,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
sql("CREATE TABLE tableOne(id int, name string, city string, age int) STORED BY 'org.apache.carbondata.format'")
sql("CREATE TABLE tableTwo(id int, age int) STORED BY 'org.apache.carbondata.format'")
@@ -244,7 +244,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "false", "false")
sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -259,7 +259,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "true", "false")
sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -274,7 +274,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("false", "false", "true")
sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -289,7 +289,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "false", "false")
sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -304,7 +304,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "true", "false")
sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -319,7 +319,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
setLoadingProperties("true", "false", "true")
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a80db8e8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
index 2cfa4c2..16e434b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
@@ -27,16 +27,9 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
override def beforeAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, "true")
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
dropTable
sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
- }
-
- override def beforeEach(): Unit = {
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
}
test("create table with no dictionary sort_columns") {
@@ -153,7 +146,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
test("filter on sort_columns include no-dictionary, direct-dictionary and dictioanry") {
sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname', 'dictionary_include'='empname')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
// no dictionary
checkAnswer(sql("select * from sorttable6 where workgroupcategory = 1"), sql("select * from origintable1 where workgroupcategory = 1 order by doj"))
// direct dictionary
@@ -166,7 +159,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("false", "false", "false")
sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -178,7 +171,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("false", "true", "false")
sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -190,7 +183,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("false", "false", "true")
sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -202,7 +195,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("true", "false", "false")
sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -214,7 +207,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("true", "true", "false")
sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
} finally {
@@ -226,7 +219,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
try {
setLoadingProperties("true", "false", "true")
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
- sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'TIMESTAMPFORMAT'='dd-MM-yyyy')""")
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
@@ -257,8 +250,6 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll with Be
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE)
- CarbonProperties.getInstance().addProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)
dropTable
}
[40/54] [abbrv] carbondata git commit: [CARBONDATA-2189] Add
DataMapProvider developer interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
new file mode 100644
index 0000000..34e11ac
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
@@ -0,0 +1,971 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.BlockMetaInfo;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
+import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.xerial.snappy.Snappy;
+
+/**
+ * Datamap implementation for blocklet.
+ */
+public class BlockletIndexDataMap extends AbstractCoarseGrainIndexDataMap implements Cacheable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletIndexDataMap.class.getName());
+
+ private static int KEY_INDEX = 0;
+
+ private static int MIN_VALUES_INDEX = 1;
+
+ private static int MAX_VALUES_INDEX = 2;
+
+ private static int ROW_COUNT_INDEX = 3;
+
+ private static int FILE_PATH_INDEX = 4;
+
+ private static int PAGE_COUNT_INDEX = 5;
+
+ private static int VERSION_INDEX = 6;
+
+ private static int SCHEMA_UPADATED_TIME_INDEX = 7;
+
+ private static int BLOCK_INFO_INDEX = 8;
+
+ private static int BLOCK_FOOTER_OFFSET = 9;
+
+ private static int LOCATIONS = 10;
+
+ private static int BLOCKLET_ID_INDEX = 11;
+
+ private static int BLOCK_LENGTH = 12;
+
+ private static int TASK_MIN_VALUES_INDEX = 0;
+
+ private static int TASK_MAX_VALUES_INDEX = 1;
+
+ private static int SCHEMA = 2;
+
+ private static int INDEX_PATH = 3;
+
+ private static int INDEX_FILE_NAME = 4;
+
+ private static int SEGMENTID = 5;
+
+ private UnsafeMemoryDMStore unsafeMemoryDMStore;
+
+ private UnsafeMemoryDMStore unsafeMemorySummaryDMStore;
+
+ private SegmentProperties segmentProperties;
+
+ private int[] columnCardinality;
+
+ @Override
+ public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
+ long startTime = System.currentTimeMillis();
+ assert (dataMapModel instanceof BlockletDataMapModel);
+ BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ List<DataFileFooter> indexInfo = fileFooterConverter
+ .getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
+ Path path = new Path(blockletDataMapInfo.getFilePath());
+ byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ byte[] segmentId =
+ blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
+ DataMapRowImpl summaryRow = null;
+ byte[] schemaBinary = null;
+ // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
+ // is id assigned to a blocklet within a part file
+ String tempFilePath = null;
+ int relativeBlockletId = 0;
+ for (DataFileFooter fileFooter : indexInfo) {
+ if (segmentProperties == null) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ schemaBinary = convertSchemaToBinary(columnInTable);
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ createSchema(segmentProperties);
+ createSummarySchema(segmentProperties, schemaBinary, filePath, fileName,
+ segmentId);
+ }
+ TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
+ BlockMetaInfo blockMetaInfo =
+ blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
+ // Here it loads info about all blocklets of index
+ // Only add if the file exists physically. There are scenarios which index file exists inside
+ // merge index but related carbondata files are deleted. In that case we first check whether
+ // the file exists physically or not
+ if (blockMetaInfo != null) {
+ if (fileFooter.getBlockletList() == null) {
+ // This is old store scenario, here blocklet information is not available in index file so
+ // load only block info
+ summaryRow =
+ loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
+ blockMetaInfo);
+ } else {
+ // blocklet ID will start from 0 again only when part file path is changed
+ if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
+ tempFilePath = blockInfo.getFilePath();
+ relativeBlockletId = 0;
+ }
+ summaryRow =
+ loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
+ blockMetaInfo, relativeBlockletId);
+ // this is done because relative blocklet id need to be incremented based on the
+ // total number of blocklets
+ relativeBlockletId += fileFooter.getBlockletList().size();
+ }
+ }
+ }
+ if (unsafeMemoryDMStore != null) {
+ unsafeMemoryDMStore.finishWriting();
+ }
+ if (null != unsafeMemorySummaryDMStore) {
+ addTaskSummaryRowToUnsafeMemoryStore(
+ summaryRow,
+ schemaBinary,
+ filePath,
+ fileName,
+ segmentId);
+ unsafeMemorySummaryDMStore.finishWriting();
+ }
+ LOGGER.info(
+ "Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (
+ System.currentTimeMillis() - startTime));
+ }
+
+ private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
+ SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
+ BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ List<BlockletInfo> blockletList = fileFooter.getBlockletList();
+ CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
+ // Add one row to maintain task level min max for segment pruning
+ if (!blockletList.isEmpty() && summaryRow == null) {
+ summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
+ }
+ for (int index = 0; index < blockletList.size(); index++) {
+ DataMapRow row = new DataMapRowImpl(schema);
+ int ordinal = 0;
+ int taskMinMaxOrdinal = 0;
+ BlockletInfo blockletInfo = blockletList.get(index);
+
+ // add start key as index key
+ row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
+
+ BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
+ byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
+ // compute and set task level min values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues,
+ TASK_MIN_VALUES_INDEX, true);
+ ordinal++;
+ taskMinMaxOrdinal++;
+ byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
+ // compute and set task level max values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
+ TASK_MAX_VALUES_INDEX, false);
+ ordinal++;
+
+ row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
+
+ // add file path
+ byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ row.setByteArray(filePathBytes, ordinal++);
+
+ // add pages
+ row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
+
+ // add version number
+ row.setShort(fileFooter.getVersionId().number(), ordinal++);
+
+ // add schema updated time
+ row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
+
+ // add blocklet info
+ byte[] serializedData;
+ try {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(stream);
+ blockletInfo.write(dataOutput);
+ serializedData = stream.toByteArray();
+ row.setByteArray(serializedData, ordinal++);
+ // Add block footer offset, it is used if we need to read footer of block
+ row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
+ ordinal++;
+ // for relative blockelt id i.e blocklet id that belongs to a particular part file
+ row.setShort((short) relativeBlockletId++, ordinal++);
+ // Store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return summaryRow;
+ }
+
+ private void setLocations(String[] locations, DataMapRow row, int ordinal)
+ throws UnsupportedEncodingException {
+ // Add location info
+ String locationStr = StringUtils.join(locations, ',');
+ row.setByteArray(locationStr.getBytes(CarbonCommonConstants.DEFAULT_CHARSET), ordinal);
+ }
+
+ /**
+ * Load information for the block.It is the case can happen only for old stores
+ * where blocklet information is not available in index file. So load only block information
+ * and read blocklet information in executor.
+ */
+ private DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter,
+ SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
+ BlockMetaInfo blockMetaInfo) {
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
+ CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
+ // Add one row to maintain task level min max for segment pruning
+ if (summaryRow == null) {
+ summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
+ }
+ DataMapRow row = new DataMapRowImpl(schema);
+ int ordinal = 0;
+ int taskMinMaxOrdinal = 0;
+ // add start key as index key
+ row.setByteArray(blockletIndex.getBtreeIndex().getStartKey(), ordinal++);
+
+ BlockletMinMaxIndex minMaxIndex = blockletIndex.getMinMaxIndex();
+ byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
+ byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
+ // update min max values in case of old store
+ byte[][] updatedMinValues =
+ CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
+ byte[][] updatedMaxValues =
+ CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMinValues), ordinal);
+ // compute and set task level min values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMinValues,
+ TASK_MIN_VALUES_INDEX, true);
+ ordinal++;
+ taskMinMaxOrdinal++;
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMaxValues), ordinal);
+ // compute and set task level max values
+ addTaskMinMaxValues(summaryRow, minMaxLen,
+ unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMaxValues,
+ TASK_MAX_VALUES_INDEX, false);
+ ordinal++;
+
+ row.setInt((int)fileFooter.getNumberOfRows(), ordinal++);
+
+ // add file path
+ byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ row.setByteArray(filePathBytes, ordinal++);
+
+ // add pages
+ row.setShort((short) 0, ordinal++);
+
+ // add version number
+ row.setShort(fileFooter.getVersionId().number(), ordinal++);
+
+ // add schema updated time
+ row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
+
+ // add blocklet info
+ row.setByteArray(new byte[0], ordinal++);
+
+ row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
+ try {
+ setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
+ ordinal++;
+ // for relative blocklet id. Value is -1 because in case of old store blocklet info will
+ // not be present in the index file and in that case we will not knwo the total number of
+ // blocklets
+ row.setShort((short) -1, ordinal++);
+
+ // store block size
+ row.setLong(blockMetaInfo.getSize(), ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return summaryRow;
+ }
+
+ private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId) {
+ // write the task summary info to unsafe memory store
+ if (null != summaryRow) {
+ // Add column schema , it is useful to generate segment properties in executor.
+ // So we no need to read footer again there.
+ if (schemaBinary != null) {
+ summaryRow.setByteArray(schemaBinary, SCHEMA);
+ }
+ summaryRow.setByteArray(filePath, INDEX_PATH);
+ summaryRow.setByteArray(fileName, INDEX_FILE_NAME);
+ summaryRow.setByteArray(segmentId, SEGMENTID);
+ try {
+ unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Fill the measures min values with minimum , this is needed for backward version compatability
+ * as older versions don't store min values for measures
+ */
+ private byte[][] updateMinValues(byte[][] minValues, int[] minMaxLen) {
+ byte[][] updatedValues = minValues;
+ if (minValues.length < minMaxLen.length) {
+ updatedValues = new byte[minMaxLen.length][];
+ System.arraycopy(minValues, 0, updatedValues, 0, minValues.length);
+ List<CarbonMeasure> measures = segmentProperties.getMeasures();
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ for (int i = 0; i < measures.size(); i++) {
+ buffer.rewind();
+ DataType dataType = measures.get(i).getDataType();
+ if (dataType == DataTypes.BYTE) {
+ buffer.putLong(Byte.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.SHORT) {
+ buffer.putLong(Short.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.INT) {
+ buffer.putLong(Integer.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.LONG) {
+ buffer.putLong(Long.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ } else if (DataTypes.isDecimal(dataType)) {
+ updatedValues[minValues.length + i] =
+ DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MIN_VALUE));
+ } else {
+ buffer.putDouble(Double.MIN_VALUE);
+ updatedValues[minValues.length + i] = buffer.array().clone();
+ }
+ }
+ }
+ return updatedValues;
+ }
+
+ /**
+ * Fill the measures max values with maximum , this is needed for backward version compatability
+ * as older versions don't store max values for measures
+ */
+ private byte[][] updateMaxValues(byte[][] maxValues, int[] minMaxLen) {
+ byte[][] updatedValues = maxValues;
+ if (maxValues.length < minMaxLen.length) {
+ updatedValues = new byte[minMaxLen.length][];
+ System.arraycopy(maxValues, 0, updatedValues, 0, maxValues.length);
+ List<CarbonMeasure> measures = segmentProperties.getMeasures();
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ for (int i = 0; i < measures.size(); i++) {
+ buffer.rewind();
+ DataType dataType = measures.get(i).getDataType();
+ if (dataType == DataTypes.BYTE) {
+ buffer.putLong(Byte.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.SHORT) {
+ buffer.putLong(Short.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.INT) {
+ buffer.putLong(Integer.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (dataType == DataTypes.LONG) {
+ buffer.putLong(Long.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ } else if (DataTypes.isDecimal(dataType)) {
+ updatedValues[maxValues.length + i] =
+ DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MAX_VALUE));
+ } else {
+ buffer.putDouble(Double.MAX_VALUE);
+ updatedValues[maxValues.length + i] = buffer.array().clone();
+ }
+ }
+ }
+ return updatedValues;
+ }
+
+ private DataMapRow addMinMax(int[] minMaxLen, CarbonRowSchema carbonRowSchema,
+ byte[][] minValues) {
+ CarbonRowSchema[] minSchemas =
+ ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
+ DataMapRow minRow = new DataMapRowImpl(minSchemas);
+ int minOrdinal = 0;
+ // min value adding
+ for (int i = 0; i < minMaxLen.length; i++) {
+ minRow.setByteArray(minValues[i], minOrdinal++);
+ }
+ return minRow;
+ }
+
+ /**
+ * This method will compute min/max values at task level
+ *
+ * @param taskMinMaxRow
+ * @param minMaxLen
+ * @param carbonRowSchema
+ * @param minMaxValue
+ * @param ordinal
+ * @param isMinValueComparison
+ */
+ private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen,
+ CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal,
+ boolean isMinValueComparison) {
+ DataMapRow row = taskMinMaxRow.getRow(ordinal);
+ byte[][] updatedMinMaxValues = minMaxValue;
+ if (null == row) {
+ CarbonRowSchema[] minSchemas =
+ ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
+ row = new DataMapRowImpl(minSchemas);
+ } else {
+ byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
+ // Compare and update min max values
+ for (int i = 0; i < minMaxLen.length; i++) {
+ int compare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
+ if (isMinValueComparison) {
+ if (compare < 0) {
+ updatedMinMaxValues[i] = existingMinMaxValues[i];
+ }
+ } else if (compare > 0) {
+ updatedMinMaxValues[i] = existingMinMaxValues[i];
+ }
+ }
+ }
+ int minMaxOrdinal = 0;
+ // min/max value adding
+ for (int i = 0; i < minMaxLen.length; i++) {
+ row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
+ }
+ taskMinMaxRow.setRow(row, ordinal);
+ }
+
+ private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
+ List<CarbonRowSchema> indexSchemas = new ArrayList<>();
+
+ // Index key
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+ getMinMaxSchema(segmentProperties, indexSchemas);
+
+ // for number of rows.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
+
+ // for table block path
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for number of pages.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for version number.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for schema updated time.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ //for blocklet info
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for block footer offset.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ // for locations
+ indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
+
+ // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
+
+ // for storing block length.
+ indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
+
+ unsafeMemoryDMStore =
+ new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
+ }
+
+ /**
+ * Creates the schema to store summary information or the information which can be stored only
+ * once per datamap. It stores datamap level max/min of each column and partition information of
+ * datamap
+ * @param segmentProperties
+ * @throws MemoryException
+ */
+ private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary,
+ byte[] filePath, byte[] fileName, byte[] segmentId)
+ throws MemoryException {
+ List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
+ getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
+ // for storing column schema
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
+ // for storing file path
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
+ // for storing file name
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
+ // for storing segmentid
+ taskMinMaxSchemas.add(
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
+ unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
+ taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
+ }
+
+ private void getMinMaxSchema(SegmentProperties segmentProperties,
+ List<CarbonRowSchema> minMaxSchemas) {
+ // Index key
+ int[] minMaxLen = segmentProperties.getColumnsValueSize();
+ // do it 2 times, one for min and one for max.
+ for (int k = 0; k < 2; k++) {
+ CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
+ for (int i = 0; i < minMaxLen.length; i++) {
+ if (minMaxLen[i] <= 0) {
+ mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
+ } else {
+ mapSchemas[i] =
+ new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
+ }
+ }
+ CarbonRowSchema mapSchema =
+ new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
+ mapSchemas);
+ minMaxSchemas.add(mapSchema);
+ }
+ }
+
+ @Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
+ boolean isScanRequired = FilterExpressionProcessor.isScanRequired(
+ filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
+ getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
+ if (isScanRequired) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
+ if (unsafeMemoryDMStore.getRowCount() == 0) {
+ return new ArrayList<>();
+ }
+ List<Blocklet> blocklets = new ArrayList<>();
+ if (filterExp == null) {
+ int rowCount = unsafeMemoryDMStore.getRowCount();
+ for (int i = 0; i < rowCount; i++) {
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
+ blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
+ }
+ } else {
+ // Remove B-tree jump logic as start and end key prepared is not
+ // correct for old store scenarios
+ int startIndex = 0;
+ int endIndex = unsafeMemoryDMStore.getRowCount();
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ while (startIndex < endIndex) {
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
+ int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
+ String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX),
+ CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ boolean isValid =
+ addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX),
+ getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
+ if (isValid) {
+ blocklets.add(createBlocklet(safeRow, blockletId));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
+ List<PartitionSpec> partitions) {
+ if (unsafeMemoryDMStore.getRowCount() == 0) {
+ return new ArrayList<>();
+ }
+ // if it has partitioned datamap but there is no partitioned information stored, it means
+ // partitions are dropped so return empty list.
+ if (partitions != null) {
+ // First get the partitions which are stored inside datamap.
+ String[] fileDetails = getFileDetails();
+ // Check the exact match of partition information inside the stored partitions.
+ boolean found = false;
+ Path folderPath = new Path(fileDetails[0]);
+ for (PartitionSpec spec : partitions) {
+ if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ return new ArrayList<>();
+ }
+ }
+ // Prune with filters if the partitions are existed in this datamap
+ return prune(filterExp, segmentProperties);
+ }
+
+ private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
+ boolean needToScan = false;
+ if (spec.getUuid() != null) {
+ String[] split = spec.getUuid().split("_");
+ if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) {
+ needToScan = true;
+ }
+ } else {
+ needToScan = true;
+ }
+ return needToScan;
+ }
+
+ /**
+ * select the blocks based on column min and max value
+ *
+ * @param filterExecuter
+ * @param maxValue
+ * @param minValue
+ * @param filePath
+ * @param blockletId
+ * @return
+ */
+ private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue,
+ byte[][] minValue, String filePath, int blockletId) {
+ BitSet bitSet = null;
+ if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+ String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
+ // this case will come in case of old store where index file does not contain the
+ // blocklet information
+ if (blockletId != -1) {
+ uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
+ }
+ bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+ .isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
+ } else {
+ bitSet = filterExecuter.isScanRequired(maxValue, minValue);
+ }
+ if (!bitSet.isEmpty()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
+ int index = Integer.parseInt(blockletId);
+ DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(index).convertToSafeRow();
+ return createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX));
+ }
+
+ private byte[][] getMinMaxValue(DataMapRow row, int index) {
+ DataMapRow minMaxRow = row.getRow(index);
+ byte[][] minMax = new byte[minMaxRow.getColumnCount()][];
+ for (int i = 0; i < minMax.length; i++) {
+ minMax[i] = minMaxRow.getByteArray(i);
+ }
+ return minMax;
+ }
+
+ private ExtendedBlocklet createBlocklet(DataMapRow row, int blockletId) {
+ ExtendedBlocklet blocklet = new ExtendedBlocklet(
+ new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
+ blockletId + "");
+ BlockletDetailInfo detailInfo = new BlockletDetailInfo();
+ detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
+ detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
+ detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
+ detailInfo.setBlockletId((short) blockletId);
+ detailInfo.setDimLens(columnCardinality);
+ detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
+ byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
+ BlockletInfo blockletInfo = null;
+ try {
+ if (byteArray.length > 0) {
+ blockletInfo = new BlockletInfo();
+ ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
+ DataInputStream inputStream = new DataInputStream(stream);
+ blockletInfo.readFields(inputStream);
+ inputStream.close();
+ }
+ blocklet.setLocation(
+ new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET)
+ .split(","));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ detailInfo.setBlockletInfo(blockletInfo);
+ blocklet.setDetailInfo(detailInfo);
+ detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
+ detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
+ detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
+ return blocklet;
+ }
+
+ private String[] getFileDetails() {
+ try {
+ String[] fileDetails = new String[3];
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ fileDetails[0] =
+ new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID),
+ CarbonCommonConstants.DEFAULT_CHARSET);
+ return fileDetails;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+
+ /**
+ * Binary search used to get the first tentative index row based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findStartIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ // if key is matched then get the first entry
+ int currentPos = mid;
+ while (currentPos - 1 >= 0
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos - 1)) == 0) {
+ currentPos--;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ // get the leaf child
+ return childNodeIndex;
+ }
+
+ /**
+ * Binary search used to get the last tentative block based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findEndIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ int currentPos = mid;
+ // if key is matched then get the first entry
+ while (currentPos + 1 < unsafeMemoryDMStore.getRowCount()
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos + 1)) == 0) {
+ currentPos++;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ return childNodeIndex;
+ }
+
+ private DataMapRow convertToRow(IndexKey key) {
+ ByteBuffer buffer =
+ ByteBuffer.allocate(key.getDictionaryKeys().length + key.getNoDictionaryKeys().length + 8);
+ buffer.putInt(key.getDictionaryKeys().length);
+ buffer.putInt(key.getNoDictionaryKeys().length);
+ buffer.put(key.getDictionaryKeys());
+ buffer.put(key.getNoDictionaryKeys());
+ DataMapRowImpl dataMapRow = new DataMapRowImpl(unsafeMemoryDMStore.getSchema());
+ dataMapRow.setByteArray(buffer.array(), 0);
+ return dataMapRow;
+ }
+
+ private byte[] getColumnSchemaBinary() {
+ DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
+ return unsafeRow.getByteArray(SCHEMA);
+ }
+
+ /**
+ * Convert schema to binary
+ */
+ private byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(stream);
+ dataOutput.writeShort(columnSchemas.size());
+ for (ColumnSchema columnSchema : columnSchemas) {
+ if (columnSchema.getColumnReferenceId() == null) {
+ columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
+ }
+ columnSchema.write(dataOutput);
+ }
+ byte[] byteArray = stream.toByteArray();
+ // Compress with snappy to reduce the size of schema
+ return Snappy.rawCompress(byteArray, byteArray.length);
+ }
+
+ @Override
+ public void clear() {
+ if (unsafeMemoryDMStore != null) {
+ unsafeMemoryDMStore.freeMemory();
+ unsafeMemoryDMStore = null;
+ segmentProperties = null;
+ }
+ // clear task min/max unsafe memory
+ if (null != unsafeMemorySummaryDMStore) {
+ unsafeMemorySummaryDMStore.freeMemory();
+ unsafeMemorySummaryDMStore = null;
+ }
+ }
+
+ @Override
+ public long getFileTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public int getAccessCount() {
+ return 0;
+ }
+
+ @Override
+ public long getMemorySize() {
+ long memoryUsed = 0L;
+ if (unsafeMemoryDMStore != null) {
+ memoryUsed += unsafeMemoryDMStore.getMemoryUsed();
+ }
+ if (null != unsafeMemorySummaryDMStore) {
+ memoryUsed += unsafeMemorySummaryDMStore.getMemoryUsed();
+ }
+ return memoryUsed;
+ }
+
+ public SegmentProperties getSegmentProperties() {
+ return segmentProperties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
new file mode 100644
index 0000000..f1b97a4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.IndexDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMapFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
+import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
+/**
+ * Table map for blocklet
+ */
+public class BlockletIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory
+ implements BlockletDetailsFetcher, SegmentPropertiesFetcher {
+
+ private static final String NAME = "clustered.btree.blocklet";
+
+ public static final DataMapSchema DATA_MAP_SCHEMA =
+ new DataMapSchema(NAME, BlockletIndexDataMapFactory.class.getName());
+
+ private AbsoluteTableIdentifier identifier;
+
+ // segmentId -> list of index file
+ private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+
+ private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainIndexDataMap> cache;
+
+ @Override
+ public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
+ this.identifier = identifier;
+ cache = CacheProvider.getInstance()
+ .createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
+ }
+
+ @Override
+ public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
+ throw new UnsupportedOperationException("not implemented");
+ }
+
+ @Override
+ public List<AbstractCoarseGrainIndexDataMap> getDataMaps(Segment segment) throws IOException {
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ return cache.getAll(tableBlockIndexUniqueIdentifiers);
+ }
+
+ private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(
+ Segment segment) throws IOException {
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ segmentMap.get(segment.getSegmentNo());
+ if (tableBlockIndexUniqueIdentifiers == null) {
+ tableBlockIndexUniqueIdentifiers = new ArrayList<>();
+ Map<String, String> indexFiles;
+ if (segment.getSegmentFileName() == null) {
+ String path =
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
+ indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ indexFiles = fileStore.getIndexFiles();
+ }
+ for (Map.Entry<String, String> indexFileEntry: indexFiles.entrySet()) {
+ Path indexFile = new Path(indexFileEntry.getKey());
+ tableBlockIndexUniqueIdentifiers.add(
+ new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(),
+ indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
+ }
+ segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
+ }
+ return tableBlockIndexUniqueIdentifiers;
+ }
+
+ /**
+ * Get the blocklet detail information based on blockletid, blockid and segmentid. This method is
+ * exclusively for BlockletIndexDataMapFactory as detail information is only available in this
+ * default datamap.
+ */
+ @Override
+ public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
+ throws IOException {
+ List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
+ // If it is already detailed blocklet then type cast and return same
+ if (blocklets.size() > 0 && blocklets.get(0) instanceof ExtendedBlocklet) {
+ for (Blocklet blocklet : blocklets) {
+ detailedBlocklets.add((ExtendedBlocklet) blocklet);
+ }
+ return detailedBlocklets;
+ }
+ List<TableBlockIndexUniqueIdentifier> identifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ // Retrieve each blocklets detail information from blocklet datamap
+ for (Blocklet blocklet : blocklets) {
+ detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
+ }
+ return detailedBlocklets;
+ }
+
+ @Override
+ public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment)
+ throws IOException {
+ if (blocklet instanceof ExtendedBlocklet) {
+ return (ExtendedBlocklet) blocklet;
+ }
+ List<TableBlockIndexUniqueIdentifier> identifiers =
+ getTableBlockIndexUniqueIdentifiers(segment);
+ return getExtendedBlocklet(identifiers, blocklet);
+ }
+
+ private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
+ Blocklet blocklet) throws IOException {
+ String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
+ for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
+ if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
+ IndexDataMap dataMap = cache.get(identifier);
+ return ((BlockletIndexDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
+ }
+ }
+ throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
+ }
+
+
+ @Override
+ public List<DataMapDistributable> toDistributable(Segment segment) {
+ List<DataMapDistributable> distributables = new ArrayList<>();
+ try {
+ CarbonFile[] carbonIndexFiles;
+ if (segment.getSegmentFileName() == null) {
+ carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
+ } else {
+ SegmentFileStore fileStore =
+ new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
+ Map<String, String> indexFiles = fileStore.getIndexFiles();
+ carbonIndexFiles = new CarbonFile[indexFiles.size()];
+ int i = 0;
+ for (String indexFile : indexFiles.keySet()) {
+ carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
+ }
+ }
+ for (int i = 0; i < carbonIndexFiles.length; i++) {
+ Path path = new Path(carbonIndexFiles[i].getPath());
+
+ FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ LocatedFileStatus fileStatus = iter.next();
+ String[] location = fileStatus.getBlockLocations()[0].getHosts();
+ BlockletDataMapDistributable distributable =
+ new BlockletDataMapDistributable(path.toString());
+ distributable.setLocations(location);
+ distributables.add(distributable);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return distributables;
+ }
+
+ @Override
+ public void fireEvent(Event event) {
+
+ }
+
+ @Override
+ public void clear(Segment segment) {
+ List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
+ if (blockIndexes != null) {
+ for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
+ IndexDataMap indexDataMap = cache.getIfPresent(blockIndex);
+ if (indexDataMap != null) {
+ cache.invalidate(blockIndex);
+ indexDataMap.clear();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void clear() {
+ for (String segmentId : segmentMap.keySet().toArray(new String[segmentMap.size()])) {
+ clear(new Segment(segmentId, null));
+ }
+ }
+
+ @Override
+ public List<AbstractCoarseGrainIndexDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
+ BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
+ List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
+ Path indexPath = new Path(mapDistributable.getFilePath());
+ String segmentNo = mapDistributable.getSegment().getSegmentNo();
+ if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
+ String parent = indexPath.getParent().toString();
+ identifiers
+ .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
+ } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
+ SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
+ CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
+ String parentPath = carbonFile.getParentFile().getAbsolutePath();
+ List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
+ for (String indexFile : indexFiles) {
+ identifiers.add(
+ new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
+ segmentNo));
+ }
+ }
+ List<AbstractCoarseGrainIndexDataMap> dataMaps;
+ try {
+ dataMaps = cache.getAll(identifiers);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return dataMaps;
+ }
+
+ @Override
+ public DataMapMeta getMeta() {
+ // TODO: pass SORT_COLUMNS into this class
+ return null;
+ }
+
+ @Override public SegmentProperties getSegmentProperties(Segment segment) throws IOException {
+ List<AbstractCoarseGrainIndexDataMap> dataMaps = getDataMaps(segment);
+ assert (dataMaps.size() > 0);
+ AbstractCoarseGrainIndexDataMap coarseGrainDataMap = dataMaps.get(0);
+ assert (coarseGrainDataMap instanceof BlockletIndexDataMap);
+ BlockletIndexDataMap dataMap = (BlockletIndexDataMap) coarseGrainDataMap;
+ return dataMap.getSegmentProperties();
+ }
+
+ @Override public List<Blocklet> getAllBlocklets(Segment segment, List<PartitionSpec> partitions)
+ throws IOException {
+ List<Blocklet> blocklets = new ArrayList<>();
+ List<AbstractCoarseGrainIndexDataMap> dataMaps = getDataMaps(segment);
+ for (AbstractCoarseGrainIndexDataMap dataMap : dataMaps) {
+ blocklets.addAll(dataMap.prune(null, getSegmentProperties(segment), partitions));
+ }
+ return blocklets;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 4d54ad5..721845d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -768,7 +768,15 @@ public class CarbonTable implements Serializable {
}
/**
- * whether this table has aggregation DataMap or not
+ * Return true if 'autoRefreshDataMap' is enabled, by default it is enabled
+ */
+ public boolean isAutoRefreshDataMap() {
+ String refresh = getTableInfo().getFactTable().getTableProperties().get("autoRefreshDataMap");
+ return refresh == null || refresh.equalsIgnoreCase("true");
+ }
+
+ /**
+ * whether this table has aggregation IndexDataMap or not
*/
public boolean hasAggregationDataMap() {
List<DataMapSchema> dataMapSchemaList = tableInfo.getDataMapSchemaList();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
index 5a9017b..ae49467 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java
@@ -32,6 +32,7 @@ public class DataMapSchema implements Serializable, Writable {
protected String dataMapName;
+ // this name can be class name of the DataMapProvider implementation or short name of it
private String className;
protected RelationIdentifier relationIdentifier;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
index d0c7386..1c6ebad 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchemaFactory.java
@@ -24,15 +24,16 @@ public class DataMapSchemaFactory {
/**
* Below class will be used to get data map schema object
* based on class name
- * @param className
+ * @param providerName
* @return data map schema
*/
- public DataMapSchema getDataMapSchema(String dataMapName, String className) {
- if (DataMapProvider.PREAGGREGATE.getClassName().equals(className) ||
- DataMapProvider.TIMESERIES.getClassName().equals(className)) {
- return new AggregationDataMapSchema(dataMapName, className);
+ public DataMapSchema getDataMapSchema(String dataMapName, String providerName) {
+ if (providerName.equalsIgnoreCase(DataMapProvider.PREAGGREGATE.toString())) {
+ return new AggregationDataMapSchema(dataMapName, providerName);
+ } else if (providerName.equalsIgnoreCase(DataMapProvider.TIMESERIES.toString())) {
+ return new AggregationDataMapSchema(dataMapName, providerName);
} else {
- return new DataMapSchema(dataMapName, className);
+ return new DataMapSchema(dataMapName, providerName);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index fff1a74..5d79abc 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -285,8 +285,7 @@ public class TableSchema implements Serializable, Writable {
// only = is allowed as special character , so replace with &
CarbonCommonConstants.DEFAULT_CHARSET)).replace("=","&"));
properties.put("QUERYTYPE", queryType);
- DataMapSchema dataMapSchema =
- new DataMapSchema(dataMapName, className);
+ DataMapSchema dataMapSchema = new DataMapSchema(dataMapName, className);
dataMapSchema.setProperties(properties);
dataMapSchema.setChildSchema(this);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
deleted file mode 100644
index 88ac3ed..0000000
--- a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestBlockletDataMap extends AbstractDictionaryCacheTest {
-
- ImplicitIncludeFilterExecutorImpl implicitIncludeFilterExecutor;
- @Before public void setUp() throws Exception {
- CarbonImplicitDimension carbonImplicitDimension =
- new CarbonImplicitDimension(0, CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID);
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
- dimColumnEvaluatorInfo.setColumnIndex(0);
- dimColumnEvaluatorInfo.setRowIndex(0);
- dimColumnEvaluatorInfo.setDimension(carbonImplicitDimension);
- dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
- implicitIncludeFilterExecutor =
- new ImplicitIncludeFilterExecutorImpl(dimColumnEvaluatorInfo);
- }
-
- @Test public void testaddBlockBasedOnMinMaxValue() throws Exception {
-
- new MockUp<ImplicitIncludeFilterExecutorImpl>() {
- @Mock BitSet isFilterValuesPresentInBlockOrBlocklet(byte[][] maxValue, byte[][] minValue,
- String uniqueBlockPath) {
- BitSet bitSet = new BitSet(1);
- bitSet.set(8);
- return bitSet;
- }
- };
-
- BlockletDataMap blockletDataMap = new BlockletDataMap();
- Method method = BlockletDataMap.class
- .getDeclaredMethod("addBlockBasedOnMinMaxValue", FilterExecuter.class, byte[][].class,
- byte[][].class, String.class, int.class);
- method.setAccessible(true);
-
- byte[][] minValue = { ByteUtil.toBytes("sfds") };
- byte[][] maxValue = { ByteUtil.toBytes("resa") };
- Object result = method
- .invoke(blockletDataMap, implicitIncludeFilterExecutor, minValue, maxValue,
- "/opt/store/default/carbon_table/Fact/Part0/Segment_0/part-0-0_batchno0-0-1514989110586.carbondata",
- 0);
- assert ((boolean) result);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
new file mode 100644
index 0000000..16048db
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
@@ -0,0 +1,59 @@
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.lang.reflect.Method;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestBlockletIndexDataMap extends AbstractDictionaryCacheTest {
+
+ ImplicitIncludeFilterExecutorImpl implicitIncludeFilterExecutor;
+ @Before public void setUp() throws Exception {
+ CarbonImplicitDimension carbonImplicitDimension =
+ new CarbonImplicitDimension(0, CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID);
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
+ dimColumnEvaluatorInfo.setColumnIndex(0);
+ dimColumnEvaluatorInfo.setRowIndex(0);
+ dimColumnEvaluatorInfo.setDimension(carbonImplicitDimension);
+ dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
+ implicitIncludeFilterExecutor =
+ new ImplicitIncludeFilterExecutorImpl(dimColumnEvaluatorInfo);
+ }
+
+ @Test public void testaddBlockBasedOnMinMaxValue() throws Exception {
+
+ new MockUp<ImplicitIncludeFilterExecutorImpl>() {
+ @Mock BitSet isFilterValuesPresentInBlockOrBlocklet(byte[][] maxValue, byte[][] minValue,
+ String uniqueBlockPath) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.set(8);
+ return bitSet;
+ }
+ };
+
+ BlockletIndexDataMap blockletDataMap = new BlockletIndexDataMap();
+ Method method = BlockletIndexDataMap.class
+ .getDeclaredMethod("addBlockBasedOnMinMaxValue", FilterExecuter.class, byte[][].class,
+ byte[][].class, String.class, int.class);
+ method.setAccessible(true);
+
+ byte[][] minValue = { ByteUtil.toBytes("sfds") };
+ byte[][] maxValue = { ByteUtil.toBytes("resa") };
+ Object result = method
+ .invoke(blockletDataMap, implicitIncludeFilterExecutor, minValue, maxValue,
+ "/opt/store/default/carbon_table/Fact/Part0/Segment_0/part-0-0_batchno0-0-1514989110586.carbondata",
+ 0);
+ assert ((boolean) result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
deleted file mode 100644
index 75dac9e..0000000
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMap.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.examples;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.row.DataMapRow;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import com.google.gson.Gson;
-
-/**
- * Datamap implementation for min max blocklet.
- */
-public class MinMaxDataMap extends AbstractCoarseGrainDataMap {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(MinMaxDataMap.class.getName());
-
- public static final String NAME = "clustered.minmax.btree.blocklet";
-
- private String filePath;
-
- private MinMaxIndexBlockDetails[] readMinMaxDataMap;
-
- @Override
- public void init(DataMapModel model) throws MemoryException, IOException {
- this.filePath = model.getFilePath();
- CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
- for (int i = 0; i < listFiles.length; i++) {
- readMinMaxDataMap = readJson(listFiles[i].getPath());
- }
- }
-
- private CarbonFile[] getCarbonMinMaxIndexFiles(String filePath, String segmentId) {
- String path = filePath.substring(0, filePath.lastIndexOf("/") + 1);
- CarbonFile carbonFile = FileFactory.getCarbonFile(path);
- return carbonFile.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().endsWith(".minmaxindex");
- }
- });
- }
-
- private MinMaxIndexBlockDetails[] readJson(String filePath) {
- Gson gsonObjectToRead = new Gson();
- DataInputStream dataInputStream = null;
- BufferedReader buffReader = null;
- InputStreamReader inStream = null;
- MinMaxIndexBlockDetails[] readMinMax = null;
- AtomicFileOperations fileOperation =
- new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
-
- try {
- if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
- return null;
- }
- dataInputStream = fileOperation.openForRead();
- inStream = new InputStreamReader(dataInputStream, "UTF-8");
- buffReader = new BufferedReader(inStream);
- readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
- } catch (IOException e) {
- return null;
- } finally {
- CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
- }
- return readMinMax;
- }
-
- /**
- * Block Prunning logic for Min Max DataMap.
- *
- * @param filterExp
- * @param segmentProperties
- * @return
- */
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp,
- SegmentProperties segmentProperties, List<PartitionSpec> partitions) {
- List<Blocklet> blocklets = new ArrayList<>();
-
- if (filterExp == null) {
- for (int i = 0; i < readMinMaxDataMap.length; i++) {
- blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
- }
- } else {
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- int startIndex = 0;
- while (startIndex < readMinMaxDataMap.length) {
- BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
- readMinMaxDataMap[startIndex].getMinValues());
- if (!bitSet.isEmpty()) {
- blocklets.add(new Blocklet(filePath,
- String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
- }
- startIndex++;
- }
- }
- return blocklets;
- }
-
- @Override
- public boolean isScanRequired(FilterResolverIntf filterExp) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void clear() {
- readMinMaxDataMap = null;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
deleted file mode 100644
index 9a67644..0000000
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataMapFactory.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.datamap.examples;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainDataMapFactory;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.events.Event;
-
-/**
- * Min Max DataMap Factory
- */
-public class MinMaxDataMapFactory extends AbstractCoarseGrainDataMapFactory {
-
- private AbsoluteTableIdentifier identifier;
-
- @Override public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
- this.identifier = identifier;
- }
-
- /**
- * createWriter will return the MinMaxDataWriter.
- *
- * @param segment
- * @return
- */
- @Override public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
- return new MinMaxDataWriter(identifier, segment, writeDirectoryPath);
- }
-
- /**
- * getDataMaps Factory method Initializes the Min Max Data Map and returns.
- *
- * @param segment
- * @return
- * @throws IOException
- */
- @Override public List<AbstractCoarseGrainDataMap> getDataMaps(Segment segment)
- throws IOException {
- List<AbstractCoarseGrainDataMap> dataMapList = new ArrayList<>();
- // Form a dataMap of Type MinMaxDataMap.
- MinMaxDataMap dataMap = new MinMaxDataMap();
- try {
- dataMap.init(new DataMapModel(
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo())));
- } catch (MemoryException ex) {
- throw new IOException(ex);
- }
- dataMapList.add(dataMap);
- return dataMapList;
- }
-
- /**
- * @param segment
- * @return
- */
- @Override public List<DataMapDistributable> toDistributable(Segment segment) {
- return null;
- }
-
- /**
- * Clear the DataMap.
- *
- * @param segment
- */
- @Override public void clear(Segment segment) {
- }
-
- /**
- * Clearing the data map.
- */
- @Override public void clear() {
- }
-
- @Override public List<AbstractCoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
- throws IOException {
- return null;
- }
-
- @Override public void fireEvent(Event event) {
-
- }
-
- @Override public DataMapMeta getMeta() {
- return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")),
- new ArrayList<ExpressionType>());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
new file mode 100644
index 0000000..8569081
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.PartitionSpec;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+/**
+ * Datamap implementation for min max blocklet.
+ */
+public class MinMaxIndexDataMap extends AbstractCoarseGrainIndexDataMap {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(MinMaxIndexDataMap.class.getName());
+
+ public static final String NAME = "clustered.minmax.btree.blocklet";
+
+ private String filePath;
+
+ private MinMaxIndexBlockDetails[] readMinMaxDataMap;
+
+ @Override
+ public void init(DataMapModel model) throws MemoryException, IOException {
+ this.filePath = model.getFilePath();
+ CarbonFile[] listFiles = getCarbonMinMaxIndexFiles(filePath, "0");
+ for (int i = 0; i < listFiles.length; i++) {
+ readMinMaxDataMap = readJson(listFiles[i].getPath());
+ }
+ }
+
+ private CarbonFile[] getCarbonMinMaxIndexFiles(String filePath, String segmentId) {
+ String path = filePath.substring(0, filePath.lastIndexOf("/") + 1);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(path);
+ return carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(".minmaxindex");
+ }
+ });
+ }
+
+ private MinMaxIndexBlockDetails[] readJson(String filePath) {
+ Gson gsonObjectToRead = new Gson();
+ DataInputStream dataInputStream = null;
+ BufferedReader buffReader = null;
+ InputStreamReader inStream = null;
+ MinMaxIndexBlockDetails[] readMinMax = null;
+ AtomicFileOperations fileOperation =
+ new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
+
+ try {
+ if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
+ return null;
+ }
+ dataInputStream = fileOperation.openForRead();
+ inStream = new InputStreamReader(dataInputStream, "UTF-8");
+ buffReader = new BufferedReader(inStream);
+ readMinMax = gsonObjectToRead.fromJson(buffReader, MinMaxIndexBlockDetails[].class);
+ } catch (IOException e) {
+ return null;
+ } finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+ }
+ return readMinMax;
+ }
+
+ /**
+ * Block Prunning logic for Min Max DataMap.
+ *
+ * @param filterExp
+ * @param segmentProperties
+ * @return
+ */
+ @Override
+ public List<Blocklet> prune(FilterResolverIntf filterExp,
+ SegmentProperties segmentProperties, List<PartitionSpec> partitions) {
+ List<Blocklet> blocklets = new ArrayList<>();
+
+ if (filterExp == null) {
+ for (int i = 0; i < readMinMaxDataMap.length; i++) {
+ blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
+ }
+ } else {
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ int startIndex = 0;
+ while (startIndex < readMinMaxDataMap.length) {
+ BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(),
+ readMinMaxDataMap[startIndex].getMinValues());
+ if (!bitSet.isEmpty()) {
+ blocklets.add(new Blocklet(filePath,
+ String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
+ }
+ startIndex++;
+ }
+ }
+ return blocklets;
+ }
+
+ @Override
+ public boolean isScanRequired(FilterResolverIntf filterExp) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clear() {
+ readMinMaxDataMap = null;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
new file mode 100644
index 0000000..9624766
--- /dev/null
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.datamap.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.datamap.DataMapDistributable;
+import org.apache.carbondata.core.datamap.DataMapMeta;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapModel;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMapFactory;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.events.Event;
+
+/**
+ * Min Max DataMap Factory
+ */
+public class MinMaxIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory {
+
+ private AbsoluteTableIdentifier identifier;
+
+ @Override public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
+ this.identifier = identifier;
+ }
+
+ /**
+ * createWriter will return the MinMaxDataWriter.
+ *
+ * @param segment
+ * @return
+ */
+ @Override public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
+ return new MinMaxDataWriter(identifier, segment, writeDirectoryPath);
+ }
+
+ /**
+ * getDataMaps Factory method Initializes the Min Max Data Map and returns.
+ *
+ * @param segment
+ * @return
+ * @throws IOException
+ */
+ @Override public List<AbstractCoarseGrainIndexDataMap> getDataMaps(Segment segment)
+ throws IOException {
+ List<AbstractCoarseGrainIndexDataMap> dataMapList = new ArrayList<>();
+ // Form a dataMap of Type MinMaxIndexDataMap.
+ MinMaxIndexDataMap dataMap = new MinMaxIndexDataMap();
+ try {
+ dataMap.init(new DataMapModel(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo())));
+ } catch (MemoryException ex) {
+ throw new IOException(ex);
+ }
+ dataMapList.add(dataMap);
+ return dataMapList;
+ }
+
+ /**
+ * @param segment
+ * @return
+ */
+ @Override public List<DataMapDistributable> toDistributable(Segment segment) {
+ return null;
+ }
+
+ /**
+ * Clear the DataMap.
+ *
+ * @param segment
+ */
+ @Override public void clear(Segment segment) {
+ }
+
+ /**
+ * Clearing the data map.
+ */
+ @Override public void clear() {
+ }
+
+ @Override public List<AbstractCoarseGrainIndexDataMap> getDataMaps(DataMapDistributable distributable)
+ throws IOException {
+ return null;
+ }
+
+ @Override public void fireEvent(Event event) {
+
+ }
+
+ @Override public DataMapMeta getMeta() {
+ return new DataMapMeta(new ArrayList<String>(Arrays.asList("c2")),
+ new ArrayList<ExpressionType>());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89a12af5/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
index 0cfe410..59872aa 100644
--- a/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
+++ b/datamap/examples/src/minmaxdatamap/test/scala/minmaxdatamaptestcase/MinMaxDataMapExample.scala
@@ -52,8 +52,8 @@ object MinMaxDataMapExample {
// register datamap writer
DataMapStoreManager.getInstance().createAndRegisterDataMap(
AbsoluteTableIdentifier.from(storeLocation, "default", "carbonminmax"),
- classOf[MinMaxDataMapFactory].getName,
- MinMaxDataMap.NAME)
+ classOf[MinMaxIndexDataMapFactory].getName,
+ MinMaxIndexDataMap.NAME)
spark.sql("DROP TABLE IF EXISTS carbonminmax")
[16/54] [abbrv] carbondata git commit: [HotFix][CheckStyle] Fix
import related checkstyle
Posted by ra...@apache.org.
[HotFix][CheckStyle] Fix import related checkstyle
This closes #1952
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4589ac53
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4589ac53
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4589ac53
Branch: refs/heads/master
Commit: 4589ac53029bd4741f95c768200f9b8202481ce8
Parents: d35fbaf
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Thu Feb 8 15:39:45 2018 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Mar 8 22:21:10 2018 +0530
----------------------------------------------------------------------
.../core/indexstore/blockletindex/BlockletDataRefNode.java | 2 +-
.../org/apache/carbondata/core/memory/HeapMemoryAllocator.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4589ac53/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
index b8fd6ff..50862a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNode.java
@@ -33,8 +33,8 @@ import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReade
import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.indexstore.FineGrainBlocklet;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* wrapper for blocklet data map data
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4589ac53/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index 53cbb1d..242995b 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -17,11 +17,11 @@
package org.apache.carbondata.core.memory;
-import javax.annotation.concurrent.GuardedBy;
import java.lang.ref.WeakReference;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
+import javax.annotation.concurrent.GuardedBy;
import org.apache.carbondata.core.util.CarbonProperties;
[46/54] [abbrv] carbondata git commit: [CARBONDATA-1543] Supported
DataMap chooser and expression for supporting multiple datamaps in single
query
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
index ccb9b68..b1962c1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggCreateCommand.scala
@@ -19,15 +19,15 @@ package org.apache.carbondata.integration.spark.testsuite.preaggregate
import scala.collection.JavaConverters._
-import org.apache.spark.sql.{AnalysisException, CarbonDatasourceHadoopRelation, Row}
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
@@ -293,7 +293,7 @@ class TestPreAggCreateCommand extends QueryTest with BeforeAndAfterAll {
test("test pre agg create table 22: using invalid datamap provider") {
sql("DROP DATAMAP IF EXISTS agg0 ON TABLE maintable")
- val e: Exception = intercept[MalformedDataMapCommandException] {
+ val e = intercept[MalformedDataMapCommandException] {
sql(
"""
| CREATE DATAMAP agg0 ON TABLE mainTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
index 43316b3..ec76b37 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala
@@ -201,7 +201,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("Unknown datamap provider/class abc"))
+ assert(e.getMessage.equals("DataMap class 'abc' not found"))
}
test("test timeseries create table 12: USING and catch MalformedCarbonCommandException") {
@@ -216,7 +216,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
- assert(e.getMessage.equals("Unknown datamap provider/class abc"))
+ assert(e.getMessage.equals("DataMap class 'abc' not found"))
}
test("test timeseries create table 13: Only one granularity level can be defined 1") {
@@ -237,6 +237,7 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
| GROUP BY dataTime
""".stripMargin)
}
+ e.printStackTrace()
assert(e.getMessage.equals("Only one granularity level can be defined"))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
index c522c1e..e31896f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSupportGlobalSortBigFileTest.scala
@@ -113,7 +113,7 @@ object CompactionSupportGlobalSortBigFileTest {
try {
val write = new PrintWriter(fileName);
for (i <- start until (start + line)) {
- write.println(i + "," + "n" + i + "," + "c" + Random.nextInt(line) + "," + Random.nextInt(80))
+ write.println(i + "," + "n" + i + "," + "c" + (i % 10000) + "," + Random.nextInt(80))
}
write.close()
} catch {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 693c145..a214444 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -92,18 +92,18 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
buildTestData
}
-test("test the boolean data type"){
- booldf.write
- .format("carbondata")
- .option("tableName", "carbon10")
- .option("tempCSV", "true")
- .option("compress", "true")
- .mode(SaveMode.Overwrite)
- .save()
- checkAnswer(
- sql("SELECT * FROM CARBON10"),
- Seq(Row("anubhav", true), Row("prince", false)))
-}
+ test("test the boolean data type"){
+ booldf.write
+ .format("carbondata")
+ .option("tableName", "carbon0")
+ .option("tempCSV", "true")
+ .option("compress", "true")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("SELECT * FROM CARBON0"),
+ Seq(Row("anubhav", true), Row("prince", false)))
+ }
test("test load dataframe with saving compressed csv files") {
// save dataframe to carbon file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index 1cbbcb4..041a63a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -36,6 +36,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
@@ -48,22 +49,21 @@ import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlo
class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
var identifier: AbsoluteTableIdentifier = _
- var dataMapName: String = _
+ var dataMapSchema: DataMapSchema = _
/**
* Initialization of Datamap factory with the identifier and datamap name
*/
- override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
this.identifier = identifier
- this.dataMapName = dataMapName
+ this.dataMapSchema = dataMapSchema
}
/**
* Return a new write for this datamap
*/
override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new CGDataMapWriter(identifier, segment, dataWritePath, dataMapName)
+ new CGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
}
/**
@@ -140,7 +140,8 @@ class CGDataMapFactory extends AbstractCoarseGrainDataMapFactory {
* Return metadata of this datamap
*/
override def getMeta: DataMapMeta = {
- new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
}
}
@@ -198,12 +199,16 @@ class CGDataMap extends AbstractCoarseGrainDataMap {
}
private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
}
- getEqualToExpression(f, buffer)
}
}
}
@@ -221,12 +226,12 @@ class CGDataMap extends AbstractCoarseGrainDataMap {
class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
segment: Segment,
dataWritePath: String,
- dataMapName: String)
+ dataMapSchema: DataMapSchema)
extends AbstractDataMapWriter(identifier, segment, dataWritePath) {
var currentBlockId: String = null
val cgwritepath = dataWritePath + "/" +
- dataMapName + System.nanoTime() + ".datamap"
+ dataMapSchema.getDataMapName + System.nanoTime() + ".datamap"
lazy val stream: DataOutputStream = FileFactory
.getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
val blockletList = new ArrayBuffer[Array[Byte]]()
@@ -345,14 +350,29 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
// register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- table.getAbsoluteTableIdentifier,
- classOf[CGDataMapFactory].getName, "cgdatamap")
+ sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
sql("select * from normal_test where name='n502670'"))
}
+ test("test cg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(s"create datamap ggdatamap1 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"create datamap ggdatamap2 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='city')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
override protected def afterAll(): Unit = {
CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
sql("DROP TABLE IF EXISTS normal_test")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index 7e93959..00d13a9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datamap.dev.cgdatamap.{AbstractCoarseGrainData
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
import org.apache.carbondata.core.scan.filter.intf.ExpressionType
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.events.Event
@@ -41,7 +42,7 @@ class C2DataMapFactory() extends AbstractCoarseGrainDataMapFactory {
var identifier: AbsoluteTableIdentifier = _
override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {
+ dataMapSchema: DataMapSchema): Unit = {
this.identifier = identifier
}
@@ -89,12 +90,9 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
}
test("test write datamap 2 pages") {
+ sql(s"CREATE TABLE carbon1(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
// register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- AbsoluteTableIdentifier.from(storeLocation + "/carbon1", "default", "carbon1"),
- classOf[C2DataMapFactory].getName,
- "test")
-
+ sql(s"CREATE DATAMAP test ON TABLE carbon1 USING '${classOf[C2DataMapFactory].getName}'")
val df = buildTestData(33000)
// save dataframe to carbon file
@@ -119,11 +117,8 @@ class DataMapWriterSuite extends QueryTest with BeforeAndAfterAll {
}
test("test write datamap 2 blocklet") {
- // register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- AbsoluteTableIdentifier.from(storeLocation + "/carbon2", "default", "carbon2"),
- classOf[C2DataMapFactory].getName,
- "test")
+ sql(s"CREATE TABLE carbon2(c1 STRING, c2 STRING, c3 INT) STORED BY 'org.apache.carbondata.format'")
+ sql(s"CREATE DATAMAP test ON TABLE carbon2 USING '${classOf[C2DataMapFactory].getName}'")
CarbonProperties.getInstance()
.addProperty("carbon.blockletgroup.size.in.mb", "1")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 9c8cc15..90f0972 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -36,6 +36,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.page.ColumnPage
import org.apache.carbondata.core.indexstore.{Blocklet, FineGrainBlocklet, PartitionSpec}
import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
@@ -48,22 +49,21 @@ import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlo
class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
var identifier: AbsoluteTableIdentifier = _
- var dataMapName: String = _
+ var dataMapSchema: DataMapSchema = _
/**
* Initialization of Datamap factory with the identifier and datamap name
*/
- override def init(identifier: AbsoluteTableIdentifier,
- dataMapName: String): Unit = {
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
this.identifier = identifier
- this.dataMapName = dataMapName
+ this.dataMapSchema = dataMapSchema
}
/**
* Return a new write for this datamap
*/
override def createWriter(segment: Segment, dataWritePath: String): AbstractDataMapWriter = {
- new FGDataMapWriter(identifier, segment, dataWritePath, dataMapName)
+ new FGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
}
/**
@@ -136,7 +136,8 @@ class FGDataMapFactory extends AbstractFineGrainDataMapFactory {
* Return metadata of this datamap
*/
override def getMeta: DataMapMeta = {
- new DataMapMeta(Seq("name").toList.asJava, new ArrayBuffer[ExpressionType]().toList.asJava)
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
}
}
@@ -225,12 +226,16 @@ class FGDataMap extends AbstractFineGrainDataMap {
}
def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
- if (expression.getChildren != null) {
- expression.getChildren.asScala.map { f =>
- if (f.isInstanceOf[EqualToExpression]) {
- buffer += f
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
}
- getEqualToExpression(f, buffer)
}
}
}
@@ -246,11 +251,12 @@ class FGDataMap extends AbstractFineGrainDataMap {
}
class FGDataMapWriter(identifier: AbsoluteTableIdentifier,
- segment: Segment, dataWriterPath: String, dataMapName: String)
+ segment: Segment, dataWriterPath: String, dataMapSchema: DataMapSchema)
extends AbstractDataMapWriter(identifier, segment, dataWriterPath) {
var currentBlockId: String = null
- val fgwritepath = dataWriterPath + "/" + System.nanoTime() + ".datamap"
+ val fgwritepath = dataWriterPath + "/" + dataMapSchema.getDataMapName + System.nanoTime() +
+ ".datamap"
val stream: DataOutputStream = FileFactory
.getDataOutputStream(fgwritepath, FileFactory.getFileType(fgwritepath))
val blockletList = new ArrayBuffer[(Array[Byte], Seq[Int], Seq[Int])]()
@@ -421,14 +427,44 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
""".stripMargin)
val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
// register datamap writer
- DataMapStoreManager.getInstance().createAndRegisterDataMap(
- table.getAbsoluteTableIdentifier,
- classOf[FGDataMapFactory].getName, "fgdatamap")
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
checkAnswer(sql("select * from datamap_test where name='n502670'"),
sql("select * from normal_test where name='n502670'"))
}
+ test("test fg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap1 ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP ggdatamap2 ON TABLE datamap_test
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('indexcolumns'='city')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
override protected def afterAll(): Unit = {
CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
sql("DROP TABLE IF EXISTS normal_test")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index 6ac9c7a..717af6f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -23,7 +23,8 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
+import org.apache.carbondata.common.exceptions.MetadataProcessException
+import org.apache.carbondata.common.exceptions.sql.{MalformedDataMapCommandException, NoSuchDataMapException}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.util.CarbonProperties
@@ -42,39 +43,22 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
val newClass = "org.apache.spark.sql.CarbonSource"
- test("test datamap create: don't support using class, only support short name") {
- intercept[MalformedDataMapCommandException] {
+ test("test datamap create: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql(s"CREATE DATAMAP datamap1 ON TABLE datamaptest USING '$newClass'")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 1)
- assert(dataMapSchemaList.get(0).getDataMapName.equals("datamap1"))
- assert(dataMapSchemaList.get(0).getClassName.equals(newClass))
}
}
- test("test datamap create with dmproperties: don't support using class") {
- intercept[MalformedDataMapCommandException] {
+ test("test datamap create with dmproperties: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql(s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
- assert(dataMapSchemaList.get(1).getDataMapName.equals("datamap2"))
- assert(dataMapSchemaList.get(1).getClassName.equals(newClass))
- assert(dataMapSchemaList.get(1).getProperties.get("key").equals("value"))
}
}
- test("test datamap create with existing name: don't support using class") {
- intercept[MalformedDataMapCommandException] {
+ test("test datamap create with existing name: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql(
s"CREATE DATAMAP datamap2 ON TABLE datamaptest USING '$newClass' DMPROPERTIES('key'='value')")
- val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
- assert(table != null)
- val dataMapSchemaList = table.getTableInfo.getDataMapSchemaList
- assert(dataMapSchemaList.size() == 2)
}
}
@@ -106,8 +90,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("drop datamap datamap_hiveMetaStoreTable on table hiveMetaStoreTable")
checkExistence(sql("show datamap on table hiveMetaStoreTable"), false, "datamap_hiveMetaStoreTable")
- }
- finally {
+ } finally {
sql("drop table hiveMetaStoreTable")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
@@ -136,6 +119,8 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
"datamap_hiveMetaStoreTable_1")
assert(sql("show datamap on table hiveMetaStoreTable_1").collect().length == 0)
sql("drop table hiveMetaStoreTable_1")
+
+ checkExistence(sql("show tables"), false, "datamap_hiveMetaStoreTable_1")
}
finally {
CarbonProperties.getInstance()
@@ -145,17 +130,17 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
test("test datamap create with preagg with duplicate name") {
- intercept[Exception] {
- sql(
- s"""
- | CREATE DATAMAP datamap2 ON TABLE datamaptest
- | USING 'preaggregate'
- | DMPROPERTIES('key'='value')
- | AS SELECT COUNT(a) FROM datamaptest
+ sql(
+ s"""
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
+ | USING 'preaggregate'
+ | DMPROPERTIES('key'='value')
+ | AS SELECT COUNT(a) FROM datamaptest
""".stripMargin)
+ intercept[MalformedDataMapCommandException] {
sql(
s"""
- | CREATE DATAMAP datamap2 ON TABLE datamaptest
+ | CREATE DATAMAP datamap10 ON TABLE datamaptest
| USING 'preaggregate'
| DMPROPERTIES('key'='value')
| AS SELECT COUNT(a) FROM datamaptest
@@ -167,10 +152,9 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
assert(dataMapSchemaList.size() == 2)
}
- test("test datamap drop with preagg") {
- intercept[Exception] {
- sql("drop table datamap3")
-
+ test("test drop non-exist datamap") {
+ intercept[NoSuchDataMapException] {
+ sql("drop datamap nonexist on table datamaptest")
}
val table = CarbonMetadata.getInstance().getCarbonTable("default", "datamaptest")
assert(table != null)
@@ -178,8 +162,8 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
assert(dataMapSchemaList.size() == 2)
}
- test("test show datamap without preaggregate: don't support using class") {
- intercept[MalformedDataMapCommandException] {
+ test("test show datamap without preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql("drop table if exists datamapshowtest")
sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
sql(s"CREATE DATAMAP datamap1 ON TABLE datamapshowtest USING '$newClass' DMPROPERTIES('key'='value')")
@@ -188,8 +172,8 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
}
}
- test("test show datamap with preaggregate: don't support using class") {
- intercept[MalformedDataMapCommandException] {
+ test("test show datamap with preaggregate: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql("drop table if exists datamapshowtest")
sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
@@ -206,8 +190,8 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
assert(sql("show datamap on table datamapshowtest").collect().length == 0)
}
- test("test show datamap after dropping datamap: don't support using class") {
- intercept[MalformedDataMapCommandException] {
+ test("test show datamap after dropping datamap: don't support using non-exist class") {
+ intercept[MetadataProcessException] {
sql("drop table if exists datamapshowtest")
sql("create table datamapshowtest (a string, b string, c string) stored by 'carbondata'")
sql("create datamap datamap1 on table datamapshowtest using 'preaggregate' as select count(a) from datamapshowtest")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/InsertOverwriteConcurrentTest.scala
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
index 60052f0..8d2f9ee 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/SparkDataMapJob.scala
@@ -37,8 +37,8 @@ import org.apache.carbondata.hadoop.api.{DataMapJob, DistributableDataMapFormat}
class SparkDataMapJob extends DataMapJob {
override def execute(dataMapFormat: DistributableDataMapFormat,
- resolverIntf: FilterResolverIntf): util.List[ExtendedBlocklet] = {
- new DataMapPruneRDD(SparkContext.getOrCreate(), dataMapFormat, resolverIntf).collect().toList
+ filter: FilterResolverIntf): util.List[ExtendedBlocklet] = {
+ new DataMapPruneRDD(SparkContext.getOrCreate(), dataMapFormat, filter).collect().toList
.asJava
}
}
@@ -53,7 +53,6 @@ class DataMapRDDPartition(rddId: Int, idx: Int, val inputSplit: InputSplit) exte
* RDD to prune the datamaps across spark cluster
* @param sc
* @param dataMapFormat
- * @param resolverIntf
*/
class DataMapPruneRDD(sc: SparkContext,
dataMapFormat: DistributableDataMapFormat,
@@ -70,7 +69,6 @@ class DataMapPruneRDD(sc: SparkContext,
val attemptId = new TaskAttemptID(jobTrackerId, id, TaskType.MAP, split.index, 0)
val attemptContext = new TaskAttemptContextImpl(new Configuration(), attemptId)
val inputSplit = split.asInstanceOf[DataMapRDDPartition].inputSplit
- DistributableDataMapFormat.setFilterExp(attemptContext.getConfiguration, resolverIntf)
val reader = dataMapFormat.createRecordReader(inputSplit, attemptContext)
reader.initialize(inputSplit, attemptContext)
val iter = new Iterator[ExtendedBlocklet] {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index adf5e04..aab2897 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -101,8 +101,12 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
CarbonException.analysisException(s"table path already exists.")
case (SaveMode.Overwrite, true) =>
val dbName = CarbonEnv.getDatabaseName(options.dbName)(sqlContext.sparkSession)
- sqlContext.sparkSession
- .sql(s"DROP TABLE IF EXISTS $dbName.${options.tableName}")
+ // In order to overwrite, delete all segments in the table
+ sqlContext.sparkSession.sql(
+ s"""
+ | DELETE FROM TABLE $dbName.${options.tableName}
+ | WHERE SEGMENT.STARTTIME BEFORE '2099-06-01 01:00:00'
+ """.stripMargin)
(true, false)
case (SaveMode.Overwrite, false) | (SaveMode.ErrorIfExists, false) =>
(true, false)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index d536746..1de66c1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -32,7 +32,9 @@ import org.apache.carbondata.core.scan.filter.intf.{ExpressionType, RowIntf}
import org.apache.carbondata.spark.util.CarbonScalaUtil
-class SparkUnknownExpression(var sparkExp: SparkExpression)
+class SparkUnknownExpression(
+ var sparkExp: SparkExpression,
+ expressionType: ExpressionType = ExpressionType.UNKNOWN)
extends UnknownExpression with ConditionalExpression {
private var evaluateExpression: (InternalRow) => Any = sparkExp.eval
@@ -64,7 +66,7 @@ class SparkUnknownExpression(var sparkExp: SparkExpression)
}
override def getFilterExpressionType: ExpressionType = {
- ExpressionType.UNKNOWN
+ expressionType
}
override def getString: String = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 6508777..c6d86b3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -16,17 +16,23 @@
*/
package org.apache.spark.sql.execution.command.datamap
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.command.preaaggregate.CreatePreAggregateTableCommand
+import org.apache.spark.sql.execution.command.preaaggregate.{CreatePreAggregateTableCommand, PreAggregateUtil}
import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
+import org.apache.spark.sql.hive.CarbonRelation
+import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
+import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider._
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
/**
* Below command class will be used to create datamap on table
@@ -52,61 +58,62 @@ case class CarbonCreateDataMapCommand(
if (carbonTable.isStreamingTable) {
throw new MalformedCarbonCommandException("Streaming table does not support creating datamap")
}
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- val dbName = tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
- val tableName = tableIdentifier.table + "_" + dataMapName
- val newDmProperties = if (dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).isDefined) {
- dmProperties.updated(TimeSeriesUtil.TIMESERIES_EVENTTIME,
- dmProperties.get(TimeSeriesUtil.TIMESERIES_EVENTTIME).get.trim)
- } else {
- dmProperties
- }
- val dataMapProvider = {
- try {
- DataMapProvider.getDataMapProvider(dmClassName)
- } catch {
- case e: UnsupportedOperationException =>
- throw new MalformedDataMapCommandException(e.getMessage)
- }
- }
- if (sparkSession.sessionState.catalog.listTables(dbName)
- .exists(_.table.equalsIgnoreCase(tableName))) {
- LOGGER.audit(
- s"Table creation with Database name [$dbName] and Table name [$tableName] failed. " +
- s"Table [$tableName] already exists under database [$dbName]")
- tableIsExists = true
- if (!ifNotExistsSet) {
- throw new TableAlreadyExistsException(dbName, tableName)
- }
- } else {
- TimeSeriesUtil.validateTimeSeriesGranularity(newDmProperties, dmClassName)
- createPreAggregateTableCommands = if (dataMapProvider == TIMESERIES) {
- val details = TimeSeriesUtil
- .getTimeSeriesGranularityDetails(newDmProperties, dmClassName)
- val updatedDmProperties = newDmProperties - details._1
+ validateDataMapName(carbonTable)
+
+ if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
+ dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ TimeSeriesUtil.validateTimeSeriesGranularity(dmProperties, dmClassName)
+ createPreAggregateTableCommands = if (dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
+ val details = TimeSeriesUtil.getTimeSeriesGranularityDetails(dmProperties, dmClassName)
+ val updatedDmProperties = dmProperties - details._1
CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
- dataMapProvider,
+ DataMapProvider.TIMESERIES,
updatedDmProperties,
queryString.get,
- Some(details._1),
- ifNotExistsSet = ifNotExistsSet)
+ Some(details._1))
} else {
CreatePreAggregateTableCommand(
dataMapName,
tableIdentifier,
- dataMapProvider,
- newDmProperties,
- queryString.get,
- ifNotExistsSet = ifNotExistsSet)
+ DataMapProvider.PREAGGREGATE,
+ dmProperties,
+ queryString.get
+ )
+ }
+ try {
+ createPreAggregateTableCommands.processMetadata(sparkSession)
+ } catch {
+ case e: Throwable => throw new MetadataProcessException(s"Failed to create datamap " +
+ s"'$dataMapName'", e)
}
- createPreAggregateTableCommands.processMetadata(sparkSession)
+ } else {
+ val dataMapSchema = new DataMapSchema(dataMapName, dmClassName)
+ dataMapSchema.setProperties(new java.util.HashMap[String, String](dmProperties.asJava))
+ val dbName = CarbonEnv.getDatabaseName(tableIdentifier.database)(sparkSession)
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.lookupRelation(
+ Some(dbName),
+ tableIdentifier.table)(sparkSession).asInstanceOf[CarbonRelation].carbonTable
+ DataMapStoreManager.getInstance().createAndRegisterDataMap(
+ carbonTable.getAbsoluteTableIdentifier, dataMapSchema)
+ // Save DataMapSchema in the schema file of main table
+ PreAggregateUtil.updateMainTable(carbonTable, dataMapSchema, sparkSession)
}
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
LOGGER.audit(s"DataMap $dataMapName successfully added to Table ${tableIdentifier.table}")
Seq.empty
}
+ private def validateDataMapName(carbonTable: CarbonTable): Unit = {
+ val existingDataMaps = carbonTable.getTableInfo.getDataMapSchemaList
+ existingDataMaps.asScala.foreach { dataMapSchema =>
+ if (dataMapSchema.getDataMapName.equalsIgnoreCase(dataMapName)) {
+ throw new MalformedDataMapCommandException(s"DataMap name '$dataMapName' already exist")
+ }
+ }
+ }
+
override def processData(sparkSession: SparkSession): Seq[Row] = {
if (dmClassName.equalsIgnoreCase(PREAGGREGATE.toString) ||
dmClassName.equalsIgnoreCase(TIMESERIES.toString)) {
@@ -116,7 +123,7 @@ case class CarbonCreateDataMapCommand(
Seq.empty
}
} else {
- throw new MalformedDataMapCommandException("Unknown datamap provider/class " + dmClassName)
+ Seq.empty
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index dcc71a2..e89d15a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.execution.command.AtomicRunnableCommand
import org.apache.spark.sql.execution.command.preaaggregate.PreAggregateUtil
import org.apache.spark.sql.execution.command.table.CarbonDropTableCommand
+import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, NoSuchDataMapException}
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.datamap.DataMapStoreManager
@@ -74,7 +75,7 @@ case class CarbonDropDataMapCommand(
Some(CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession))
} catch {
case ex: NoSuchTableException =>
- throw ex
+ throw new MetadataProcessException(s"Dropping datamap $dataMapName failed", ex)
}
// If datamap to be dropped in parent table then drop the datamap from metastore and remove
// entry from parent table.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
index c836584..c02ac4f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/CreatePreAggregateTableCommand.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.execution.command.timeseries.TimeSeriesUtil
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
+import org.apache.carbondata.common.exceptions.sql.MalformedDataMapCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema
@@ -67,8 +68,10 @@ case class CreatePreAggregateTableCommand(
dmProperties.foreach(t => tableProperties.put(t._1, t._2))
parentTable = PreAggregateUtil.getParentCarbonTable(df.logicalPlan)
- assert(parentTable.getTableName.equalsIgnoreCase(parentTableIdentifier.table),
- "Parent table name is different in select and create")
+ if (!parentTable.getTableName.equalsIgnoreCase(parentTableIdentifier.table)) {
+ throw new MalformedDataMapCommandException(
+ "Parent table name is different in select and create")
+ }
var neworder = Seq[String]()
val parentOrder = parentTable.getSortColumns(parentTable.getTableName).asScala
parentOrder.foreach(parentcol =>
@@ -131,23 +134,19 @@ case class CreatePreAggregateTableCommand(
dmProperties.foreach(f => childSchema.getProperties.put(f._1, f._2))
// updating the parent table about child table
- try {
- PreAggregateUtil.updateMainTable(
- CarbonEnv.getDatabaseName(parentTableIdentifier.database)(sparkSession),
- parentTableIdentifier.table,
- childSchema,
- sparkSession)
- } catch {
- case ex: Exception =>
- undoMetadata(sparkSession, ex)
- throw ex
- }
+ PreAggregateUtil.updateMainTable(
+ parentTable,
+ childSchema,
+ sparkSession)
+ // After updating the parent carbon table with data map entry extract the latest table object
+ // to be used in further create process.
+ parentTable = CarbonEnv.getCarbonTable(parentTableIdentifier.database,
+ parentTableIdentifier.table)(sparkSession)
val updatedLoadQuery = if (timeSeriesFunction.isDefined) {
PreAggregateUtil.createTimeSeriesSelectQueryFromMain(childSchema.getChildSchema,
parentTable.getTableName,
parentTable.getDatabaseName)
- }
- else {
+ } else {
queryString
}
val dataFrame = sparkSession.sql(new CarbonSpark2SqlParser().addPreAggLoadFunction(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index a5d256c..dd42b50 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.spark.sql.types.DataType
+import org.apache.carbondata.common.exceptions.MetadataProcessException
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -406,21 +407,20 @@ object PreAggregateUtil {
* Below method will be used to update the main table about the pre aggregate table information
* in case of any exception it will throw error so pre aggregate table creation will fail
*
- * @param dbName
- * @param tableName
* @param childSchema
* @param sparkSession
*/
- def updateMainTable(dbName: String, tableName: String,
+ def updateMainTable(carbonTable: CarbonTable,
childSchema: DataMapSchema, sparkSession: SparkSession): Unit = {
val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val locksToBeAcquired = List(LockUsage.METADATA_LOCK,
LockUsage.DROP_TABLE_LOCK)
var locks = List.empty[ICarbonLock]
- var carbonTable: CarbonTable = null
+ var numberOfCurrentChild: Int = 0
+ val dbName = carbonTable.getDatabaseName
+ val tableName = carbonTable.getTableName
try {
val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
- carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
locks = acquireLock(dbName, tableName, locksToBeAcquired, carbonTable)
// get the latest carbon table and check for column existence
// read the latest schema file
@@ -433,7 +433,7 @@ object PreAggregateUtil {
carbonTable.getTablePath)
if (wrapperTableInfo.getDataMapSchemaList.asScala.
exists(f => f.getDataMapName.equalsIgnoreCase(childSchema.getDataMapName))) {
- throw new Exception("Duplicate datamap")
+ throw new MetadataProcessException("DataMap name already exist")
}
wrapperTableInfo.getDataMapSchemaList.add(childSchema)
val thriftTable = schemaConverter
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
index f6d5e25..e82c9d7 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/strategy/CarbonLateDecodeStrategy.scala
@@ -614,7 +614,7 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case StartsWith(a: Attribute, Literal(v, t)) =>
+ case s@StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
case c@EndsWith(a: Attribute, Literal(v, t)) =>
Some(CarbonEndsWith(c))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
index e5eb53c..38c5146 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -41,6 +41,10 @@ import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonCol
import org.apache.carbondata.core.scan.expression.conditional._
import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.core.util.{CarbonProperties, ThreadLocalSessionInfo}
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.util.ThreadLocalSessionInfo
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -50,6 +54,7 @@ import org.apache.carbondata.spark.util.CarbonScalaUtil
*/
object CarbonFilters {
+ val carbonProperties = CarbonProperties.getInstance()
/**
* Converts data sources filters to carbon filter predicates.
*/
@@ -117,25 +122,20 @@ object CarbonFilters {
new OrExpression(lhsFilter, rhsFilter)
}
case sources.StringStartsWith(name, value) if value.length > 0 =>
- val l = new GreaterThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value))
- val maxValueLimit = value.substring(0, value.length - 1) +
- (value.charAt(value.length - 1).toInt + 1).toChar
- val r = new LessThanExpression(
- getCarbonExpression(name), getCarbonLiteralExpression(name, maxValueLimit))
- Some(new AndExpression(l, r))
+ Some(new StartsWithExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
case CarbonEndsWith(expr: Expression) =>
Some(new SparkUnknownExpression(expr.transform {
case AttributeReference(name, dataType, _, _) =>
CarbonBoundReference(new CarbonColumnExpression(name.toString,
CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
- }))
+ }, ExpressionType.ENDSWITH))
case CarbonContainsWith(expr: Expression) =>
Some(new SparkUnknownExpression(expr.transform {
case AttributeReference(name, dataType, _, _) =>
CarbonBoundReference(new CarbonColumnExpression(name.toString,
CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
- }))
+ }, ExpressionType.CONTAINSWITH))
case CastExpr(expr: Expression) =>
Some(transformExpression(expr))
case FalseExpr() =>
@@ -266,7 +266,7 @@ object CarbonFilters {
CastExpressionOptimization.checkIfCastCanBeRemove(c)
case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
CastExpressionOptimization.checkIfCastCanBeRemove(c)
- case StartsWith(a: Attribute, Literal(v, t)) =>
+ case s@StartsWith(a: Attribute, Literal(v, t)) =>
Some(sources.StringStartsWith(a.name, v.toString))
case c@EndsWith(a: Attribute, Literal(v, t)) =>
Some(CarbonEndsWith(c))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
index 66f8bc5..f514074 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datamap/DataMapWriterListener.java
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.processing.store.TablePage;
/**
@@ -50,9 +50,9 @@ public class DataMapWriterListener {
/**
* register all datamap writer for specified table and segment
*/
- public void registerAllWriter(AbsoluteTableIdentifier identifier, String segmentId,
+ public void registerAllWriter(CarbonTable carbonTable, String segmentId,
String dataWritePath) {
- List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(identifier);
+ List<TableDataMap> tableDataMaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable);
if (tableDataMaps != null) {
for (TableDataMap tableDataMap : tableDataMaps) {
DataMapFactory factory = tableDataMap.getDataMapFactory();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
index 0ee77da..21a4b3e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadProcessBuilder.java
@@ -263,7 +263,7 @@ public final class DataLoadProcessBuilder {
if (carbonTable.isHivePartitionTable()) {
configuration.setWritingCoresCount((short) 1);
}
- TableSpec tableSpec = new TableSpec(dimensions, measures);
+ TableSpec tableSpec = new TableSpec(carbonTable);
configuration.setTableSpec(tableSpec);
return configuration;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/56330ae2/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d6af747..8aa5bde 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -260,8 +260,8 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.sortScope = CarbonDataProcessorUtil.getSortScope(configuration);
DataMapWriterListener listener = new DataMapWriterListener();
- listener.registerAllWriter(configuration.getTableIdentifier(), configuration.getSegmentId(),
- storeLocation[new Random().nextInt(storeLocation.length)]);
+ listener.registerAllWriter(configuration.getTableSpec().getCarbonTable(),
+ configuration.getSegmentId(), storeLocation[new Random().nextInt(storeLocation.length)]);
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
carbonFactDataHandlerModel.writingCoresCount = configuration.getWritingCoresCount();
@@ -320,13 +320,11 @@ public class CarbonFactDataHandlerModel {
carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
- carbonFactDataHandlerModel.tableSpec = new TableSpec(
- segmentProperties.getDimensions(),
- segmentProperties.getMeasures());
-
+ carbonFactDataHandlerModel.tableSpec =
+ new TableSpec(loadModel.getCarbonDataLoadSchema().getCarbonTable());
DataMapWriterListener listener = new DataMapWriterListener();
listener.registerAllWriter(
- loadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(),
+ loadModel.getCarbonDataLoadSchema().getCarbonTable(),
loadModel.getSegmentId(),
tempStoreLocation[new Random().nextInt(tempStoreLocation.length)]);
carbonFactDataHandlerModel.dataMapWriterlistener = listener;
[51/54] [abbrv] carbondata git commit: [HOTFIX] Add dava doc for
datamap interface
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
deleted file mode 100644
index 34e11ac..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMap.java
+++ /dev/null
@@ -1,971 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cacheable;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.indexstore.BlockMetaInfo;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
-import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
-import org.apache.carbondata.core.indexstore.row.DataMapRow;
-import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
-import org.apache.carbondata.core.indexstore.schema.CarbonRowSchema;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataFileFooterConverter;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.xerial.snappy.Snappy;
-
-/**
- * Datamap implementation for blocklet.
- */
-public class BlockletIndexDataMap extends AbstractCoarseGrainIndexDataMap implements Cacheable {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockletIndexDataMap.class.getName());
-
- private static int KEY_INDEX = 0;
-
- private static int MIN_VALUES_INDEX = 1;
-
- private static int MAX_VALUES_INDEX = 2;
-
- private static int ROW_COUNT_INDEX = 3;
-
- private static int FILE_PATH_INDEX = 4;
-
- private static int PAGE_COUNT_INDEX = 5;
-
- private static int VERSION_INDEX = 6;
-
- private static int SCHEMA_UPADATED_TIME_INDEX = 7;
-
- private static int BLOCK_INFO_INDEX = 8;
-
- private static int BLOCK_FOOTER_OFFSET = 9;
-
- private static int LOCATIONS = 10;
-
- private static int BLOCKLET_ID_INDEX = 11;
-
- private static int BLOCK_LENGTH = 12;
-
- private static int TASK_MIN_VALUES_INDEX = 0;
-
- private static int TASK_MAX_VALUES_INDEX = 1;
-
- private static int SCHEMA = 2;
-
- private static int INDEX_PATH = 3;
-
- private static int INDEX_FILE_NAME = 4;
-
- private static int SEGMENTID = 5;
-
- private UnsafeMemoryDMStore unsafeMemoryDMStore;
-
- private UnsafeMemoryDMStore unsafeMemorySummaryDMStore;
-
- private SegmentProperties segmentProperties;
-
- private int[] columnCardinality;
-
- @Override
- public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
- long startTime = System.currentTimeMillis();
- assert (dataMapModel instanceof BlockletDataMapModel);
- BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
- DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- List<DataFileFooter> indexInfo = fileFooterConverter
- .getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
- Path path = new Path(blockletDataMapInfo.getFilePath());
- byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- byte[] segmentId =
- blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
- DataMapRowImpl summaryRow = null;
- byte[] schemaBinary = null;
- // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
- // is id assigned to a blocklet within a part file
- String tempFilePath = null;
- int relativeBlockletId = 0;
- for (DataFileFooter fileFooter : indexInfo) {
- if (segmentProperties == null) {
- List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
- schemaBinary = convertSchemaToBinary(columnInTable);
- columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
- segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
- createSchema(segmentProperties);
- createSummarySchema(segmentProperties, schemaBinary, filePath, fileName,
- segmentId);
- }
- TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
- BlockMetaInfo blockMetaInfo =
- blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
- // Here it loads info about all blocklets of index
- // Only add if the file exists physically. There are scenarios which index file exists inside
- // merge index but related carbondata files are deleted. In that case we first check whether
- // the file exists physically or not
- if (blockMetaInfo != null) {
- if (fileFooter.getBlockletList() == null) {
- // This is old store scenario, here blocklet information is not available in index file so
- // load only block info
- summaryRow =
- loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- blockMetaInfo);
- } else {
- // blocklet ID will start from 0 again only when part file path is changed
- if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
- tempFilePath = blockInfo.getFilePath();
- relativeBlockletId = 0;
- }
- summaryRow =
- loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow,
- blockMetaInfo, relativeBlockletId);
- // this is done because relative blocklet id need to be incremented based on the
- // total number of blocklets
- relativeBlockletId += fileFooter.getBlockletList().size();
- }
- }
- }
- if (unsafeMemoryDMStore != null) {
- unsafeMemoryDMStore.finishWriting();
- }
- if (null != unsafeMemorySummaryDMStore) {
- addTaskSummaryRowToUnsafeMemoryStore(
- summaryRow,
- schemaBinary,
- filePath,
- fileName,
- segmentId);
- unsafeMemorySummaryDMStore.finishWriting();
- }
- LOGGER.info(
- "Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (
- System.currentTimeMillis() - startTime));
- }
-
- private DataMapRowImpl loadToUnsafe(DataFileFooter fileFooter,
- SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- BlockMetaInfo blockMetaInfo, int relativeBlockletId) {
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- List<BlockletInfo> blockletList = fileFooter.getBlockletList();
- CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
- // Add one row to maintain task level min max for segment pruning
- if (!blockletList.isEmpty() && summaryRow == null) {
- summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
- }
- for (int index = 0; index < blockletList.size(); index++) {
- DataMapRow row = new DataMapRowImpl(schema);
- int ordinal = 0;
- int taskMinMaxOrdinal = 0;
- BlockletInfo blockletInfo = blockletList.get(index);
-
- // add start key as index key
- row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
-
- BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
- byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], minValues), ordinal);
- // compute and set task level min values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], minValues,
- TASK_MIN_VALUES_INDEX, true);
- ordinal++;
- taskMinMaxOrdinal++;
- byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], maxValues), ordinal);
- // compute and set task level max values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], maxValues,
- TASK_MAX_VALUES_INDEX, false);
- ordinal++;
-
- row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
-
- // add file path
- byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- row.setByteArray(filePathBytes, ordinal++);
-
- // add pages
- row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
-
- // add version number
- row.setShort(fileFooter.getVersionId().number(), ordinal++);
-
- // add schema updated time
- row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
-
- // add blocklet info
- byte[] serializedData;
- try {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- DataOutput dataOutput = new DataOutputStream(stream);
- blockletInfo.write(dataOutput);
- serializedData = stream.toByteArray();
- row.setByteArray(serializedData, ordinal++);
- // Add block footer offset, it is used if we need to read footer of block
- row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
- setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
- ordinal++;
- // for relative blockelt id i.e blocklet id that belongs to a particular part file
- row.setShort((short) relativeBlockletId++, ordinal++);
- // Store block size
- row.setLong(blockMetaInfo.getSize(), ordinal);
- unsafeMemoryDMStore.addIndexRowToUnsafe(row);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- return summaryRow;
- }
-
- private void setLocations(String[] locations, DataMapRow row, int ordinal)
- throws UnsupportedEncodingException {
- // Add location info
- String locationStr = StringUtils.join(locations, ',');
- row.setByteArray(locationStr.getBytes(CarbonCommonConstants.DEFAULT_CHARSET), ordinal);
- }
-
- /**
- * Load information for the block.It is the case can happen only for old stores
- * where blocklet information is not available in index file. So load only block information
- * and read blocklet information in executor.
- */
- private DataMapRowImpl loadToUnsafeBlock(DataFileFooter fileFooter,
- SegmentProperties segmentProperties, String filePath, DataMapRowImpl summaryRow,
- BlockMetaInfo blockMetaInfo) {
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
- CarbonRowSchema[] schema = unsafeMemoryDMStore.getSchema();
- // Add one row to maintain task level min max for segment pruning
- if (summaryRow == null) {
- summaryRow = new DataMapRowImpl(unsafeMemorySummaryDMStore.getSchema());
- }
- DataMapRow row = new DataMapRowImpl(schema);
- int ordinal = 0;
- int taskMinMaxOrdinal = 0;
- // add start key as index key
- row.setByteArray(blockletIndex.getBtreeIndex().getStartKey(), ordinal++);
-
- BlockletMinMaxIndex minMaxIndex = blockletIndex.getMinMaxIndex();
- byte[][] minValues = updateMinValues(minMaxIndex.getMinValues(), minMaxLen);
- byte[][] maxValues = updateMaxValues(minMaxIndex.getMaxValues(), minMaxLen);
- // update min max values in case of old store
- byte[][] updatedMinValues =
- CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
- byte[][] updatedMaxValues =
- CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
- row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMinValues), ordinal);
- // compute and set task level min values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMinValues,
- TASK_MIN_VALUES_INDEX, true);
- ordinal++;
- taskMinMaxOrdinal++;
- row.setRow(addMinMax(minMaxLen, schema[ordinal], updatedMaxValues), ordinal);
- // compute and set task level max values
- addTaskMinMaxValues(summaryRow, minMaxLen,
- unsafeMemorySummaryDMStore.getSchema()[taskMinMaxOrdinal], updatedMaxValues,
- TASK_MAX_VALUES_INDEX, false);
- ordinal++;
-
- row.setInt((int)fileFooter.getNumberOfRows(), ordinal++);
-
- // add file path
- byte[] filePathBytes = filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- row.setByteArray(filePathBytes, ordinal++);
-
- // add pages
- row.setShort((short) 0, ordinal++);
-
- // add version number
- row.setShort(fileFooter.getVersionId().number(), ordinal++);
-
- // add schema updated time
- row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
-
- // add blocklet info
- row.setByteArray(new byte[0], ordinal++);
-
- row.setLong(fileFooter.getBlockInfo().getTableBlockInfo().getBlockOffset(), ordinal++);
- try {
- setLocations(blockMetaInfo.getLocationInfo(), row, ordinal);
- ordinal++;
- // for relative blocklet id. Value is -1 because in case of old store blocklet info will
- // not be present in the index file and in that case we will not knwo the total number of
- // blocklets
- row.setShort((short) -1, ordinal++);
-
- // store block size
- row.setLong(blockMetaInfo.getSize(), ordinal);
- unsafeMemoryDMStore.addIndexRowToUnsafe(row);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- return summaryRow;
- }
-
- private void addTaskSummaryRowToUnsafeMemoryStore(DataMapRow summaryRow, byte[] schemaBinary,
- byte[] filePath, byte[] fileName, byte[] segmentId) {
- // write the task summary info to unsafe memory store
- if (null != summaryRow) {
- // Add column schema , it is useful to generate segment properties in executor.
- // So we no need to read footer again there.
- if (schemaBinary != null) {
- summaryRow.setByteArray(schemaBinary, SCHEMA);
- }
- summaryRow.setByteArray(filePath, INDEX_PATH);
- summaryRow.setByteArray(fileName, INDEX_FILE_NAME);
- summaryRow.setByteArray(segmentId, SEGMENTID);
- try {
- unsafeMemorySummaryDMStore.addIndexRowToUnsafe(summaryRow);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- /**
- * Fill the measures min values with minimum , this is needed for backward version compatability
- * as older versions don't store min values for measures
- */
- private byte[][] updateMinValues(byte[][] minValues, int[] minMaxLen) {
- byte[][] updatedValues = minValues;
- if (minValues.length < minMaxLen.length) {
- updatedValues = new byte[minMaxLen.length][];
- System.arraycopy(minValues, 0, updatedValues, 0, minValues.length);
- List<CarbonMeasure> measures = segmentProperties.getMeasures();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- for (int i = 0; i < measures.size(); i++) {
- buffer.rewind();
- DataType dataType = measures.get(i).getDataType();
- if (dataType == DataTypes.BYTE) {
- buffer.putLong(Byte.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.SHORT) {
- buffer.putLong(Short.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.INT) {
- buffer.putLong(Integer.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.LONG) {
- buffer.putLong(Long.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- } else if (DataTypes.isDecimal(dataType)) {
- updatedValues[minValues.length + i] =
- DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MIN_VALUE));
- } else {
- buffer.putDouble(Double.MIN_VALUE);
- updatedValues[minValues.length + i] = buffer.array().clone();
- }
- }
- }
- return updatedValues;
- }
-
- /**
- * Fill the measures max values with maximum , this is needed for backward version compatability
- * as older versions don't store max values for measures
- */
- private byte[][] updateMaxValues(byte[][] maxValues, int[] minMaxLen) {
- byte[][] updatedValues = maxValues;
- if (maxValues.length < minMaxLen.length) {
- updatedValues = new byte[minMaxLen.length][];
- System.arraycopy(maxValues, 0, updatedValues, 0, maxValues.length);
- List<CarbonMeasure> measures = segmentProperties.getMeasures();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- for (int i = 0; i < measures.size(); i++) {
- buffer.rewind();
- DataType dataType = measures.get(i).getDataType();
- if (dataType == DataTypes.BYTE) {
- buffer.putLong(Byte.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.SHORT) {
- buffer.putLong(Short.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.INT) {
- buffer.putLong(Integer.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (dataType == DataTypes.LONG) {
- buffer.putLong(Long.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- } else if (DataTypes.isDecimal(dataType)) {
- updatedValues[maxValues.length + i] =
- DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MAX_VALUE));
- } else {
- buffer.putDouble(Double.MAX_VALUE);
- updatedValues[maxValues.length + i] = buffer.array().clone();
- }
- }
- }
- return updatedValues;
- }
-
- private DataMapRow addMinMax(int[] minMaxLen, CarbonRowSchema carbonRowSchema,
- byte[][] minValues) {
- CarbonRowSchema[] minSchemas =
- ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
- DataMapRow minRow = new DataMapRowImpl(minSchemas);
- int minOrdinal = 0;
- // min value adding
- for (int i = 0; i < minMaxLen.length; i++) {
- minRow.setByteArray(minValues[i], minOrdinal++);
- }
- return minRow;
- }
-
- /**
- * This method will compute min/max values at task level
- *
- * @param taskMinMaxRow
- * @param minMaxLen
- * @param carbonRowSchema
- * @param minMaxValue
- * @param ordinal
- * @param isMinValueComparison
- */
- private void addTaskMinMaxValues(DataMapRow taskMinMaxRow, int[] minMaxLen,
- CarbonRowSchema carbonRowSchema, byte[][] minMaxValue, int ordinal,
- boolean isMinValueComparison) {
- DataMapRow row = taskMinMaxRow.getRow(ordinal);
- byte[][] updatedMinMaxValues = minMaxValue;
- if (null == row) {
- CarbonRowSchema[] minSchemas =
- ((CarbonRowSchema.StructCarbonRowSchema) carbonRowSchema).getChildSchemas();
- row = new DataMapRowImpl(minSchemas);
- } else {
- byte[][] existingMinMaxValues = getMinMaxValue(taskMinMaxRow, ordinal);
- // Compare and update min max values
- for (int i = 0; i < minMaxLen.length; i++) {
- int compare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(existingMinMaxValues[i], minMaxValue[i]);
- if (isMinValueComparison) {
- if (compare < 0) {
- updatedMinMaxValues[i] = existingMinMaxValues[i];
- }
- } else if (compare > 0) {
- updatedMinMaxValues[i] = existingMinMaxValues[i];
- }
- }
- }
- int minMaxOrdinal = 0;
- // min/max value adding
- for (int i = 0; i < minMaxLen.length; i++) {
- row.setByteArray(updatedMinMaxValues[i], minMaxOrdinal++);
- }
- taskMinMaxRow.setRow(row, ordinal);
- }
-
- private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
- List<CarbonRowSchema> indexSchemas = new ArrayList<>();
-
- // Index key
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
- getMinMaxSchema(segmentProperties, indexSchemas);
-
- // for number of rows.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
-
- // for table block path
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for number of pages.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for version number.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for schema updated time.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- //for blocklet info
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for block footer offset.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- // for locations
- indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
-
- // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
-
- // for storing block length.
- indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
-
- unsafeMemoryDMStore =
- new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
- }
-
- /**
- * Creates the schema to store summary information or the information which can be stored only
- * once per datamap. It stores datamap level max/min of each column and partition information of
- * datamap
- * @param segmentProperties
- * @throws MemoryException
- */
- private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary,
- byte[] filePath, byte[] fileName, byte[] segmentId)
- throws MemoryException {
- List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
- getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
- // for storing column schema
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
- // for storing file path
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
- // for storing file name
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
- // for storing segmentid
- taskMinMaxSchemas.add(
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
- unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(
- taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
- }
-
- private void getMinMaxSchema(SegmentProperties segmentProperties,
- List<CarbonRowSchema> minMaxSchemas) {
- // Index key
- int[] minMaxLen = segmentProperties.getColumnsValueSize();
- // do it 2 times, one for min and one for max.
- for (int k = 0; k < 2; k++) {
- CarbonRowSchema[] mapSchemas = new CarbonRowSchema[minMaxLen.length];
- for (int i = 0; i < minMaxLen.length; i++) {
- if (minMaxLen[i] <= 0) {
- mapSchemas[i] = new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY);
- } else {
- mapSchemas[i] =
- new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
- }
- }
- CarbonRowSchema mapSchema =
- new CarbonRowSchema.StructCarbonRowSchema(DataTypes.createDefaultStructType(),
- mapSchemas);
- minMaxSchemas.add(mapSchema);
- }
- }
-
- @Override
- public boolean isScanRequired(FilterResolverIntf filterExp) {
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- for (int i = 0; i < unsafeMemorySummaryDMStore.getRowCount(); i++) {
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(i);
- boolean isScanRequired = FilterExpressionProcessor.isScanRequired(
- filterExecuter, getMinMaxValue(unsafeRow, TASK_MAX_VALUES_INDEX),
- getMinMaxValue(unsafeRow, TASK_MIN_VALUES_INDEX));
- if (isScanRequired) {
- return true;
- }
- }
- return false;
- }
-
- private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
- if (unsafeMemoryDMStore.getRowCount() == 0) {
- return new ArrayList<>();
- }
- List<Blocklet> blocklets = new ArrayList<>();
- if (filterExp == null) {
- int rowCount = unsafeMemoryDMStore.getRowCount();
- for (int i = 0; i < rowCount; i++) {
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
- blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
- }
- } else {
- // Remove B-tree jump logic as start and end key prepared is not
- // correct for old store scenarios
- int startIndex = 0;
- int endIndex = unsafeMemoryDMStore.getRowCount();
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
- while (startIndex < endIndex) {
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
- int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
- String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX),
- CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
- boolean isValid =
- addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX),
- getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
- if (isValid) {
- blocklets.add(createBlocklet(safeRow, blockletId));
- }
- startIndex++;
- }
- }
- return blocklets;
- }
-
- @Override
- public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
- List<PartitionSpec> partitions) {
- if (unsafeMemoryDMStore.getRowCount() == 0) {
- return new ArrayList<>();
- }
- // if it has partitioned datamap but there is no partitioned information stored, it means
- // partitions are dropped so return empty list.
- if (partitions != null) {
- // First get the partitions which are stored inside datamap.
- String[] fileDetails = getFileDetails();
- // Check the exact match of partition information inside the stored partitions.
- boolean found = false;
- Path folderPath = new Path(fileDetails[0]);
- for (PartitionSpec spec : partitions) {
- if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) {
- found = true;
- break;
- }
- }
- if (!found) {
- return new ArrayList<>();
- }
- }
- // Prune with filters if the partitions are existed in this datamap
- return prune(filterExp, segmentProperties);
- }
-
- private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {
- boolean needToScan = false;
- if (spec.getUuid() != null) {
- String[] split = spec.getUuid().split("_");
- if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil
- .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) {
- needToScan = true;
- }
- } else {
- needToScan = true;
- }
- return needToScan;
- }
-
- /**
- * select the blocks based on column min and max value
- *
- * @param filterExecuter
- * @param maxValue
- * @param minValue
- * @param filePath
- * @param blockletId
- * @return
- */
- private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue,
- byte[][] minValue, String filePath, int blockletId) {
- BitSet bitSet = null;
- if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
- String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
- // this case will come in case of old store where index file does not contain the
- // blocklet information
- if (blockletId != -1) {
- uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
- }
- bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
- .isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
- } else {
- bitSet = filterExecuter.isScanRequired(maxValue, minValue);
- }
- if (!bitSet.isEmpty()) {
- return true;
- } else {
- return false;
- }
- }
-
- public ExtendedBlocklet getDetailedBlocklet(String blockletId) {
- int index = Integer.parseInt(blockletId);
- DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(index).convertToSafeRow();
- return createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX));
- }
-
- private byte[][] getMinMaxValue(DataMapRow row, int index) {
- DataMapRow minMaxRow = row.getRow(index);
- byte[][] minMax = new byte[minMaxRow.getColumnCount()][];
- for (int i = 0; i < minMax.length; i++) {
- minMax[i] = minMaxRow.getByteArray(i);
- }
- return minMax;
- }
-
- private ExtendedBlocklet createBlocklet(DataMapRow row, int blockletId) {
- ExtendedBlocklet blocklet = new ExtendedBlocklet(
- new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
- blockletId + "");
- BlockletDetailInfo detailInfo = new BlockletDetailInfo();
- detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
- detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
- detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
- detailInfo.setBlockletId((short) blockletId);
- detailInfo.setDimLens(columnCardinality);
- detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
- byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
- BlockletInfo blockletInfo = null;
- try {
- if (byteArray.length > 0) {
- blockletInfo = new BlockletInfo();
- ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
- DataInputStream inputStream = new DataInputStream(stream);
- blockletInfo.readFields(inputStream);
- inputStream.close();
- }
- blocklet.setLocation(
- new String(row.getByteArray(LOCATIONS), CarbonCommonConstants.DEFAULT_CHARSET)
- .split(","));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- detailInfo.setBlockletInfo(blockletInfo);
- blocklet.setDetailInfo(detailInfo);
- detailInfo.setBlockFooterOffset(row.getLong(BLOCK_FOOTER_OFFSET));
- detailInfo.setColumnSchemaBinary(getColumnSchemaBinary());
- detailInfo.setBlockSize(row.getLong(BLOCK_LENGTH));
- return blocklet;
- }
-
- private String[] getFileDetails() {
- try {
- String[] fileDetails = new String[3];
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
- fileDetails[0] =
- new String(unsafeRow.getByteArray(INDEX_PATH), CarbonCommonConstants.DEFAULT_CHARSET);
- fileDetails[1] = new String(unsafeRow.getByteArray(INDEX_FILE_NAME),
- CarbonCommonConstants.DEFAULT_CHARSET);
- fileDetails[2] = new String(unsafeRow.getByteArray(SEGMENTID),
- CarbonCommonConstants.DEFAULT_CHARSET);
- return fileDetails;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
-
- /**
- * Binary search used to get the first tentative index row based on
- * search key
- *
- * @param key search key
- * @return first tentative block
- */
- private int findStartIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
- int childNodeIndex;
- int low = 0;
- int high = unsafeMemoryDMStore.getRowCount() - 1;
- int mid = 0;
- int compareRes = -1;
- //
- while (low <= high) {
- mid = (low + high) >>> 1;
- // compare the entries
- compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
- if (compareRes < 0) {
- high = mid - 1;
- } else if (compareRes > 0) {
- low = mid + 1;
- } else {
- // if key is matched then get the first entry
- int currentPos = mid;
- while (currentPos - 1 >= 0
- && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos - 1)) == 0) {
- currentPos--;
- }
- mid = currentPos;
- break;
- }
- }
- // if compare result is less than zero then we
- // and mid is more than 0 then we need to previous block as duplicates
- // record can be present
- if (compareRes < 0) {
- if (mid > 0) {
- mid--;
- }
- childNodeIndex = mid;
- } else {
- childNodeIndex = mid;
- }
- // get the leaf child
- return childNodeIndex;
- }
-
- /**
- * Binary search used to get the last tentative block based on
- * search key
- *
- * @param key search key
- * @return first tentative block
- */
- private int findEndIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
- int childNodeIndex;
- int low = 0;
- int high = unsafeMemoryDMStore.getRowCount() - 1;
- int mid = 0;
- int compareRes = -1;
- //
- while (low <= high) {
- mid = (low + high) >>> 1;
- // compare the entries
- compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
- if (compareRes < 0) {
- high = mid - 1;
- } else if (compareRes > 0) {
- low = mid + 1;
- } else {
- int currentPos = mid;
- // if key is matched then get the first entry
- while (currentPos + 1 < unsafeMemoryDMStore.getRowCount()
- && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos + 1)) == 0) {
- currentPos++;
- }
- mid = currentPos;
- break;
- }
- }
- // if compare result is less than zero then we
- // and mid is more than 0 then we need to previous block as duplicates
- // record can be present
- if (compareRes < 0) {
- if (mid > 0) {
- mid--;
- }
- childNodeIndex = mid;
- } else {
- childNodeIndex = mid;
- }
- return childNodeIndex;
- }
-
- private DataMapRow convertToRow(IndexKey key) {
- ByteBuffer buffer =
- ByteBuffer.allocate(key.getDictionaryKeys().length + key.getNoDictionaryKeys().length + 8);
- buffer.putInt(key.getDictionaryKeys().length);
- buffer.putInt(key.getNoDictionaryKeys().length);
- buffer.put(key.getDictionaryKeys());
- buffer.put(key.getNoDictionaryKeys());
- DataMapRowImpl dataMapRow = new DataMapRowImpl(unsafeMemoryDMStore.getSchema());
- dataMapRow.setByteArray(buffer.array(), 0);
- return dataMapRow;
- }
-
- private byte[] getColumnSchemaBinary() {
- DataMapRow unsafeRow = unsafeMemorySummaryDMStore.getUnsafeRow(0);
- return unsafeRow.getByteArray(SCHEMA);
- }
-
- /**
- * Convert schema to binary
- */
- private byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException {
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- DataOutput dataOutput = new DataOutputStream(stream);
- dataOutput.writeShort(columnSchemas.size());
- for (ColumnSchema columnSchema : columnSchemas) {
- if (columnSchema.getColumnReferenceId() == null) {
- columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
- }
- columnSchema.write(dataOutput);
- }
- byte[] byteArray = stream.toByteArray();
- // Compress with snappy to reduce the size of schema
- return Snappy.rawCompress(byteArray, byteArray.length);
- }
-
- @Override
- public void clear() {
- if (unsafeMemoryDMStore != null) {
- unsafeMemoryDMStore.freeMemory();
- unsafeMemoryDMStore = null;
- segmentProperties = null;
- }
- // clear task min/max unsafe memory
- if (null != unsafeMemorySummaryDMStore) {
- unsafeMemorySummaryDMStore.freeMemory();
- unsafeMemorySummaryDMStore = null;
- }
- }
-
- @Override
- public long getFileTimeStamp() {
- return 0;
- }
-
- @Override
- public int getAccessCount() {
- return 0;
- }
-
- @Override
- public long getMemorySize() {
- long memoryUsed = 0L;
- if (unsafeMemoryDMStore != null) {
- memoryUsed += unsafeMemoryDMStore.getMemoryUsed();
- }
- if (null != unsafeMemorySummaryDMStore) {
- memoryUsed += unsafeMemorySummaryDMStore.getMemoryUsed();
- }
- return memoryUsed;
- }
-
- public SegmentProperties getSegmentProperties() {
- return segmentProperties;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
deleted file mode 100644
index f1b97a4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletIndexDataMapFactory.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datamap.DataMapDistributable;
-import org.apache.carbondata.core.datamap.DataMapMeta;
-import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
-import org.apache.carbondata.core.datamap.dev.IndexDataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMapFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
-import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
-import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
-import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.SegmentFileStore;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.events.Event;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-
-/**
- * Table map for blocklet
- */
-public class BlockletIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory
- implements BlockletDetailsFetcher, SegmentPropertiesFetcher {
-
- private static final String NAME = "clustered.btree.blocklet";
-
- public static final DataMapSchema DATA_MAP_SCHEMA =
- new DataMapSchema(NAME, BlockletIndexDataMapFactory.class.getName());
-
- private AbsoluteTableIdentifier identifier;
-
- // segmentId -> list of index file
- private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
-
- private Cache<TableBlockIndexUniqueIdentifier, AbstractCoarseGrainIndexDataMap> cache;
-
- @Override
- public void init(AbsoluteTableIdentifier identifier, DataMapSchema dataMapSchema) {
- this.identifier = identifier;
- cache = CacheProvider.getInstance()
- .createCache(CacheType.DRIVER_BLOCKLET_DATAMAP);
- }
-
- @Override
- public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public List<AbstractCoarseGrainIndexDataMap> getDataMaps(Segment segment) throws IOException {
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- return cache.getAll(tableBlockIndexUniqueIdentifiers);
- }
-
- private List<TableBlockIndexUniqueIdentifier> getTableBlockIndexUniqueIdentifiers(
- Segment segment) throws IOException {
- List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
- segmentMap.get(segment.getSegmentNo());
- if (tableBlockIndexUniqueIdentifiers == null) {
- tableBlockIndexUniqueIdentifiers = new ArrayList<>();
- Map<String, String> indexFiles;
- if (segment.getSegmentFileName() == null) {
- String path =
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo());
- indexFiles = new SegmentIndexFileStore().getIndexFilesFromSegment(path);
- } else {
- SegmentFileStore fileStore =
- new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
- indexFiles = fileStore.getIndexFiles();
- }
- for (Map.Entry<String, String> indexFileEntry: indexFiles.entrySet()) {
- Path indexFile = new Path(indexFileEntry.getKey());
- tableBlockIndexUniqueIdentifiers.add(
- new TableBlockIndexUniqueIdentifier(indexFile.getParent().toString(),
- indexFile.getName(), indexFileEntry.getValue(), segment.getSegmentNo()));
- }
- segmentMap.put(segment.getSegmentNo(), tableBlockIndexUniqueIdentifiers);
- }
- return tableBlockIndexUniqueIdentifiers;
- }
-
- /**
- * Get the blocklet detail information based on blockletid, blockid and segmentid. This method is
- * exclusively for BlockletIndexDataMapFactory as detail information is only available in this
- * default datamap.
- */
- @Override
- public List<ExtendedBlocklet> getExtendedBlocklets(List<Blocklet> blocklets, Segment segment)
- throws IOException {
- List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
- // If it is already detailed blocklet then type cast and return same
- if (blocklets.size() > 0 && blocklets.get(0) instanceof ExtendedBlocklet) {
- for (Blocklet blocklet : blocklets) {
- detailedBlocklets.add((ExtendedBlocklet) blocklet);
- }
- return detailedBlocklets;
- }
- List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- // Retrieve each blocklets detail information from blocklet datamap
- for (Blocklet blocklet : blocklets) {
- detailedBlocklets.add(getExtendedBlocklet(identifiers, blocklet));
- }
- return detailedBlocklets;
- }
-
- @Override
- public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment)
- throws IOException {
- if (blocklet instanceof ExtendedBlocklet) {
- return (ExtendedBlocklet) blocklet;
- }
- List<TableBlockIndexUniqueIdentifier> identifiers =
- getTableBlockIndexUniqueIdentifiers(segment);
- return getExtendedBlocklet(identifiers, blocklet);
- }
-
- private ExtendedBlocklet getExtendedBlocklet(List<TableBlockIndexUniqueIdentifier> identifiers,
- Blocklet blocklet) throws IOException {
- String carbonIndexFileName = CarbonTablePath.getCarbonIndexFileName(blocklet.getBlockId());
- for (TableBlockIndexUniqueIdentifier identifier : identifiers) {
- if (identifier.getIndexFilePath().equals(carbonIndexFileName)) {
- IndexDataMap dataMap = cache.get(identifier);
- return ((BlockletIndexDataMap) dataMap).getDetailedBlocklet(blocklet.getBlockletId());
- }
- }
- throw new IOException("Blocklet with blockid " + blocklet.getBlockletId() + " not found ");
- }
-
-
- @Override
- public List<DataMapDistributable> toDistributable(Segment segment) {
- List<DataMapDistributable> distributables = new ArrayList<>();
- try {
- CarbonFile[] carbonIndexFiles;
- if (segment.getSegmentFileName() == null) {
- carbonIndexFiles = SegmentIndexFileStore.getCarbonIndexFiles(
- CarbonTablePath.getSegmentPath(identifier.getTablePath(), segment.getSegmentNo()));
- } else {
- SegmentFileStore fileStore =
- new SegmentFileStore(identifier.getTablePath(), segment.getSegmentFileName());
- Map<String, String> indexFiles = fileStore.getIndexFiles();
- carbonIndexFiles = new CarbonFile[indexFiles.size()];
- int i = 0;
- for (String indexFile : indexFiles.keySet()) {
- carbonIndexFiles[i++] = FileFactory.getCarbonFile(indexFile);
- }
- }
- for (int i = 0; i < carbonIndexFiles.length; i++) {
- Path path = new Path(carbonIndexFiles[i].getPath());
-
- FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
- LocatedFileStatus fileStatus = iter.next();
- String[] location = fileStatus.getBlockLocations()[0].getHosts();
- BlockletDataMapDistributable distributable =
- new BlockletDataMapDistributable(path.toString());
- distributable.setLocations(location);
- distributables.add(distributable);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return distributables;
- }
-
- @Override
- public void fireEvent(Event event) {
-
- }
-
- @Override
- public void clear(Segment segment) {
- List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segment.getSegmentNo());
- if (blockIndexes != null) {
- for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
- IndexDataMap indexDataMap = cache.getIfPresent(blockIndex);
- if (indexDataMap != null) {
- cache.invalidate(blockIndex);
- indexDataMap.clear();
- }
- }
- }
- }
-
- @Override
- public void clear() {
- for (String segmentId : segmentMap.keySet().toArray(new String[segmentMap.size()])) {
- clear(new Segment(segmentId, null));
- }
- }
-
- @Override
- public List<AbstractCoarseGrainIndexDataMap> getDataMaps(DataMapDistributable distributable)
- throws IOException {
- BlockletDataMapDistributable mapDistributable = (BlockletDataMapDistributable) distributable;
- List<TableBlockIndexUniqueIdentifier> identifiers = new ArrayList<>();
- Path indexPath = new Path(mapDistributable.getFilePath());
- String segmentNo = mapDistributable.getSegment().getSegmentNo();
- if (indexPath.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
- String parent = indexPath.getParent().toString();
- identifiers
- .add(new TableBlockIndexUniqueIdentifier(parent, indexPath.getName(), null, segmentNo));
- } else if (indexPath.getName().endsWith(CarbonTablePath.MERGE_INDEX_FILE_EXT)) {
- SegmentIndexFileStore fileStore = new SegmentIndexFileStore();
- CarbonFile carbonFile = FileFactory.getCarbonFile(indexPath.toString());
- String parentPath = carbonFile.getParentFile().getAbsolutePath();
- List<String> indexFiles = fileStore.getIndexFilesFromMergeFile(carbonFile.getAbsolutePath());
- for (String indexFile : indexFiles) {
- identifiers.add(
- new TableBlockIndexUniqueIdentifier(parentPath, indexFile, carbonFile.getName(),
- segmentNo));
- }
- }
- List<AbstractCoarseGrainIndexDataMap> dataMaps;
- try {
- dataMaps = cache.getAll(identifiers);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return dataMaps;
- }
-
- @Override
- public DataMapMeta getMeta() {
- // TODO: pass SORT_COLUMNS into this class
- return null;
- }
-
- @Override public SegmentProperties getSegmentProperties(Segment segment) throws IOException {
- List<AbstractCoarseGrainIndexDataMap> dataMaps = getDataMaps(segment);
- assert (dataMaps.size() > 0);
- AbstractCoarseGrainIndexDataMap coarseGrainDataMap = dataMaps.get(0);
- assert (coarseGrainDataMap instanceof BlockletIndexDataMap);
- BlockletIndexDataMap dataMap = (BlockletIndexDataMap) coarseGrainDataMap;
- return dataMap.getSegmentProperties();
- }
-
- @Override public List<Blocklet> getAllBlocklets(Segment segment, List<PartitionSpec> partitions)
- throws IOException {
- List<Blocklet> blocklets = new ArrayList<>();
- List<AbstractCoarseGrainIndexDataMap> dataMaps = getDataMaps(segment);
- for (AbstractCoarseGrainIndexDataMap dataMap : dataMaps) {
- blocklets.addAll(dataMap.prune(null, getSegmentProperties(segment), partitions));
- }
- return blocklets;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 721845d..f81f1dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -776,7 +776,7 @@ public class CarbonTable implements Serializable {
}
/**
- * whether this table has aggregation IndexDataMap or not
+ * whether this table has aggregation DataMap or not
*/
public boolean hasAggregationDataMap() {
List<DataMapSchema> dataMapSchemaList = tableInfo.getDataMapSchemaList();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
new file mode 100644
index 0000000..fa142aa
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
@@ -0,0 +1,59 @@
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.lang.reflect.Method;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestBlockletDataMap extends AbstractDictionaryCacheTest {
+
+ ImplicitIncludeFilterExecutorImpl implicitIncludeFilterExecutor;
+ @Before public void setUp() throws Exception {
+ CarbonImplicitDimension carbonImplicitDimension =
+ new CarbonImplicitDimension(0, CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID);
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
+ dimColumnEvaluatorInfo.setColumnIndex(0);
+ dimColumnEvaluatorInfo.setRowIndex(0);
+ dimColumnEvaluatorInfo.setDimension(carbonImplicitDimension);
+ dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
+ implicitIncludeFilterExecutor =
+ new ImplicitIncludeFilterExecutorImpl(dimColumnEvaluatorInfo);
+ }
+
+ @Test public void testaddBlockBasedOnMinMaxValue() throws Exception {
+
+ new MockUp<ImplicitIncludeFilterExecutorImpl>() {
+ @Mock BitSet isFilterValuesPresentInBlockOrBlocklet(byte[][] maxValue, byte[][] minValue,
+ String uniqueBlockPath) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.set(8);
+ return bitSet;
+ }
+ };
+
+ BlockletDataMap blockletDataMap = new BlockletDataMap();
+ Method method = BlockletDataMap.class
+ .getDeclaredMethod("addBlockBasedOnMinMaxValue", FilterExecuter.class, byte[][].class,
+ byte[][].class, String.class, int.class);
+ method.setAccessible(true);
+
+ byte[][] minValue = { ByteUtil.toBytes("sfds") };
+ byte[][] maxValue = { ByteUtil.toBytes("resa") };
+ Object result = method
+ .invoke(blockletDataMap, implicitIncludeFilterExecutor, minValue, maxValue,
+ "/opt/store/default/carbon_table/Fact/Part0/Segment_0/part-0-0_batchno0-0-1514989110586.carbondata",
+ 0);
+ assert ((boolean) result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
deleted file mode 100644
index 16048db..0000000
--- a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletIndexDataMap.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.apache.carbondata.core.indexstore.blockletindex;
-
-import java.lang.reflect.Method;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
-import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestBlockletIndexDataMap extends AbstractDictionaryCacheTest {
-
- ImplicitIncludeFilterExecutorImpl implicitIncludeFilterExecutor;
- @Before public void setUp() throws Exception {
- CarbonImplicitDimension carbonImplicitDimension =
- new CarbonImplicitDimension(0, CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID);
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
- dimColumnEvaluatorInfo.setColumnIndex(0);
- dimColumnEvaluatorInfo.setRowIndex(0);
- dimColumnEvaluatorInfo.setDimension(carbonImplicitDimension);
- dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
- implicitIncludeFilterExecutor =
- new ImplicitIncludeFilterExecutorImpl(dimColumnEvaluatorInfo);
- }
-
- @Test public void testaddBlockBasedOnMinMaxValue() throws Exception {
-
- new MockUp<ImplicitIncludeFilterExecutorImpl>() {
- @Mock BitSet isFilterValuesPresentInBlockOrBlocklet(byte[][] maxValue, byte[][] minValue,
- String uniqueBlockPath) {
- BitSet bitSet = new BitSet(1);
- bitSet.set(8);
- return bitSet;
- }
- };
-
- BlockletIndexDataMap blockletDataMap = new BlockletIndexDataMap();
- Method method = BlockletIndexDataMap.class
- .getDeclaredMethod("addBlockBasedOnMinMaxValue", FilterExecuter.class, byte[][].class,
- byte[][].class, String.class, int.class);
- method.setAccessible(true);
-
- byte[][] minValue = { ByteUtil.toBytes("sfds") };
- byte[][] maxValue = { ByteUtil.toBytes("resa") };
- Object result = method
- .invoke(blockletDataMap, implicitIncludeFilterExecutor, minValue, maxValue,
- "/opt/store/default/carbon_table/Fact/Part0/Segment_0/part-0-0_batchno0-0-1514989110586.carbondata",
- 0);
- assert ((boolean) result);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
index 17c8332..e116825 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxDataWriter.java
@@ -29,7 +29,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.Segment;
-import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
-public class MinMaxDataWriter extends AbstractDataMapWriter {
+public class MinMaxDataWriter extends DataMapWriter {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
index 8569081..bad22b2 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMap.java
@@ -28,7 +28,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
@@ -49,7 +49,7 @@ import com.google.gson.Gson;
/**
* Datamap implementation for min max blocklet.
*/
-public class MinMaxIndexDataMap extends AbstractCoarseGrainIndexDataMap {
+public class MinMaxIndexDataMap extends CoarseGrainDataMap {
private static final LogService LOGGER =
LogServiceFactory.getLogService(MinMaxIndexDataMap.class.getName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
index 9624766..9fea55b 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
@@ -27,8 +27,9 @@ import org.apache.carbondata.core.datamap.DataMapMeta;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.dev.AbstractDataMapWriter;
import org.apache.carbondata.core.datamap.dev.DataMapModel;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMap;
-import org.apache.carbondata.core.datamap.dev.cgdatamap.AbstractCoarseGrainIndexDataMapFactory;
+import org.apache.carbondata.core.datamap.dev.DataMapWriter;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMap;
+import org.apache.carbondata.core.datamap.dev.cgdatamap.CoarseGrainDataMapFactory;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
@@ -39,7 +40,7 @@ import org.apache.carbondata.events.Event;
/**
* Min Max DataMap Factory
*/
-public class MinMaxIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFactory {
+public class MinMaxIndexDataMapFactory extends CoarseGrainDataMapFactory {
private AbsoluteTableIdentifier identifier;
@@ -53,7 +54,7 @@ public class MinMaxIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFa
* @param segment
* @return
*/
- @Override public AbstractDataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
+ @Override public DataMapWriter createWriter(Segment segment, String writeDirectoryPath) {
return new MinMaxDataWriter(identifier, segment, writeDirectoryPath);
}
@@ -64,9 +65,9 @@ public class MinMaxIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFa
* @return
* @throws IOException
*/
- @Override public List<AbstractCoarseGrainIndexDataMap> getDataMaps(Segment segment)
+ @Override public List<CoarseGrainDataMap> getDataMaps(Segment segment)
throws IOException {
- List<AbstractCoarseGrainIndexDataMap> dataMapList = new ArrayList<>();
+ List<CoarseGrainDataMap> dataMapList = new ArrayList<>();
// Form a dataMap of Type MinMaxIndexDataMap.
MinMaxIndexDataMap dataMap = new MinMaxIndexDataMap();
try {
@@ -101,7 +102,7 @@ public class MinMaxIndexDataMapFactory extends AbstractCoarseGrainIndexDataMapFa
@Override public void clear() {
}
- @Override public List<AbstractCoarseGrainIndexDataMap> getDataMaps(DataMapDistributable distributable)
+ @Override public List<CoarseGrainDataMap> getDataMaps(DataMapDistributable distributable)
throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 007ba2f..81b5fde 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -33,8 +33,8 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapChooser;
+import org.apache.carbondata.core.datamap.DataMapLevel;
import org.apache.carbondata.core.datamap.DataMapStoreManager;
-import org.apache.carbondata.core.datamap.DataMapType;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
@@ -43,7 +43,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.exception.InvalidConfigurationException;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.indexstore.blockletindex.BlockletIndexDataMapFactory;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
@@ -751,11 +751,11 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
List<ExtendedBlocklet> prunedBlocklets;
- if (distributedCG || dataMapExprWrapper.getDataMapType() == DataMapType.FG) {
+ if (distributedCG || dataMapExprWrapper.getDataMapType() == DataMapLevel.FG) {
DistributableDataMapFormat datamapDstr =
new DistributableDataMapFormat(absoluteTableIdentifier, dataMapExprWrapper,
segmentIds, partitionsToPrune,
- BlockletIndexDataMapFactory.class.getName());
+ BlockletDataMapFactory.class.getName());
prunedBlocklets = dataMapJob.execute(datamapDstr, resolver);
// Apply expression on the blocklets.
prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/fc2a7eb3/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
new file mode 100644
index 0000000..3cba3ff
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.datamap
+
+import java.io.{ByteArrayInputStream, DataOutputStream, ObjectInputStream, ObjectOutputStream}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, Segment}
+import org.apache.carbondata.core.datamap.dev.{DataMapModel, DataMapWriter}
+import org.apache.carbondata.core.datamap.dev.cgdatamap.{CoarseGrainDataMap, CoarseGrainDataMapFactory}
+import org.apache.carbondata.core.datastore.FileReader
+import org.apache.carbondata.core.datastore.block.SegmentProperties
+import org.apache.carbondata.core.datastore.compression.SnappyCompressor
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.page.ColumnPage
+import org.apache.carbondata.core.indexstore.{Blocklet, PartitionSpec}
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapDistributable
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf
+import org.apache.carbondata.core.util.ByteUtil
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.events.Event
+import org.apache.carbondata.spark.testsuite.datacompaction.CompactionSupportGlobalSortBigFileTest
+
+class CGDataMapFactory extends CoarseGrainDataMapFactory {
+ var identifier: AbsoluteTableIdentifier = _
+ var dataMapSchema: DataMapSchema = _
+
+ /**
+ * Initialization of Datamap factory with the identifier and datamap name
+ */
+ override def init(identifier: AbsoluteTableIdentifier, dataMapSchema: DataMapSchema): Unit = {
+ this.identifier = identifier
+ this.dataMapSchema = dataMapSchema
+ }
+
+ /**
+ * Return a new write for this datamap
+ */
+ override def createWriter(segment: Segment, dataWritePath: String): DataMapWriter = {
+ new CGDataMapWriter(identifier, segment, dataWritePath, dataMapSchema)
+ }
+
+ /**
+ * Get the datamap for segmentid
+ */
+ override def getDataMaps(segment: Segment): java.util.List[CoarseGrainDataMap] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map {f =>
+ val dataMap: CoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(f.getCanonicalPath))
+ dataMap
+ }.toList.asJava
+ }
+
+
+ /**
+ * Get datamaps for distributable object.
+ */
+ override def getDataMaps(
+ distributable: DataMapDistributable): java.util.List[CoarseGrainDataMap] = {
+ val mapDistributable = distributable.asInstanceOf[BlockletDataMapDistributable]
+ val dataMap: CoarseGrainDataMap = new CGDataMap()
+ dataMap.init(new DataMapModel(mapDistributable.getFilePath))
+ Seq(dataMap).asJava
+ }
+
+ /**
+ *
+ * @param event
+ */
+ override def fireEvent(event: Event): Unit = {
+ ???
+ }
+
+ /**
+ * Get all distributable objects of a segmentid
+ *
+ * @return
+ */
+ override def toDistributable(segment: Segment): java.util.List[DataMapDistributable] = {
+ val file = FileFactory.getCarbonFile(
+ CarbonTablePath.getSegmentPath(identifier.getTablePath, segment.getSegmentNo))
+
+ val files = file.listFiles(new CarbonFileFilter {
+ override def accept(file: CarbonFile): Boolean = file.getName.endsWith(".datamap")
+ })
+ files.map { f =>
+ val d:DataMapDistributable = new BlockletDataMapDistributable(f.getCanonicalPath)
+ d
+ }.toList.asJava
+ }
+
+
+ /**
+ * Clears datamap of the segment
+ */
+ override def clear(segment: Segment): Unit = {
+
+ }
+
+ /**
+ * Clear all datamaps from memory
+ */
+ override def clear(): Unit = {
+
+ }
+
+ /**
+ * Return metadata of this datamap
+ */
+ override def getMeta: DataMapMeta = {
+ new DataMapMeta(dataMapSchema.getProperties.get("indexcolumns").split(",").toList.asJava,
+ List(ExpressionType.EQUALS, ExpressionType.IN).asJava)
+ }
+}
+
+class CGDataMap extends CoarseGrainDataMap {
+
+ var maxMin: ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))] = _
+ var FileReader: FileReader = _
+ var filePath: String = _
+ val compressor = new SnappyCompressor
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ override def init(dataMapModel: DataMapModel): Unit = {
+ this.filePath = dataMapModel.getFilePath
+ val size = FileFactory.getCarbonFile(filePath).getSize
+ FileReader = FileFactory.getFileHolder(FileFactory.getFileType(filePath))
+ val footerLen = FileReader.readInt(filePath, size-4)
+ val bytes = FileReader.readByteArray(filePath, size-footerLen-4, footerLen)
+ val in = new ByteArrayInputStream(compressor.unCompressByte(bytes))
+ val obj = new ObjectInputStream(in)
+ maxMin = obj.readObject().asInstanceOf[ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]]
+ }
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ override def prune(
+ filterExp: FilterResolverIntf,
+ segmentProperties: SegmentProperties,
+ partitions: java.util.List[PartitionSpec]): java.util.List[Blocklet] = {
+ val buffer: ArrayBuffer[Expression] = new ArrayBuffer[Expression]()
+ val expression = filterExp.getFilterExpression
+ getEqualToExpression(expression, buffer)
+ val value = buffer.map { f =>
+ f.getChildren.get(1).evaluate(null).getString
+ }
+ val meta = findMeta(value(0).getBytes)
+ meta.map { f=>
+ new Blocklet(f._1, f._2 + "")
+ }.asJava
+ }
+
+
+ private def findMeta(value: Array[Byte]) = {
+ val tuples = maxMin.filter { f =>
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._1) <= 0 &&
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(value, f._3._2) >= 0
+ }
+ tuples
+ }
+
+ private def getEqualToExpression(expression: Expression, buffer: ArrayBuffer[Expression]): Unit = {
+ if (expression.isInstanceOf[EqualToExpression]) {
+ buffer += expression
+ } else {
+ if (expression.getChildren != null) {
+ expression.getChildren.asScala.map { f =>
+ if (f.isInstanceOf[EqualToExpression]) {
+ buffer += f
+ }
+ getEqualToExpression(f, buffer)
+ }
+ }
+ }
+ }
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ override def clear() = {
+ ???
+ }
+
+ override def isScanRequired(filterExp: FilterResolverIntf): Boolean = ???
+}
+
+class CGDataMapWriter(identifier: AbsoluteTableIdentifier,
+ segment: Segment,
+ dataWritePath: String,
+ dataMapSchema: DataMapSchema)
+ extends DataMapWriter(identifier, segment, dataWritePath) {
+
+ var currentBlockId: String = null
+ val cgwritepath = dataWritePath + "/" +
+ dataMapSchema.getDataMapName + System.nanoTime() + ".datamap"
+ lazy val stream: DataOutputStream = FileFactory
+ .getDataOutputStream(cgwritepath, FileFactory.getFileType(cgwritepath))
+ val blockletList = new ArrayBuffer[Array[Byte]]()
+ val maxMin = new ArrayBuffer[(String, Int, (Array[Byte], Array[Byte]))]()
+ val compressor = new SnappyCompressor
+
+ /**
+ * Start of new block notification.
+ *
+ * @param blockId file name of the carbondata file
+ */
+ override def onBlockStart(blockId: String): Unit = {
+ currentBlockId = blockId
+ }
+
+ /**
+ * End of block notification
+ */
+ override def onBlockEnd(blockId: String): Unit = {
+
+ }
+
+ /**
+ * Start of new blocklet notification.
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletStart(blockletId: Int): Unit = {
+
+ }
+
+ /**
+ * End of blocklet notification
+ *
+ * @param blockletId sequence number of blocklet in the block
+ */
+ override def onBlockletEnd(blockletId: Int): Unit = {
+ val sorted = blockletList
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ maxMin +=
+ ((currentBlockId+"", blockletId, (sorted.last, sorted.head)))
+ blockletList.clear()
+ }
+
+ /**
+ * Add the column pages row to the datamap, order of pages is same as `indexColumns` in
+ * DataMapMeta returned in DataMapFactory.
+ *
+ * Implementation should copy the content of `pages` as needed, because `pages` memory
+ * may be freed after this method returns, if using unsafe column page.
+ */
+ override def onPageAdded(blockletId: Int,
+ pageId: Int,
+ pages: Array[ColumnPage]): Unit = {
+ val size = pages(0).getPageSize
+ val list = new ArrayBuffer[Array[Byte]]()
+ var i = 0
+ while (i < size) {
+ val bytes = pages(0).getBytes(i)
+ val newBytes = new Array[Byte](bytes.length - 2)
+ System.arraycopy(bytes, 2, newBytes, 0, newBytes.length)
+ list += newBytes
+ i = i + 1
+ }
+ // Sort based on the column data in order to create index.
+ val sorted = list
+ .sortWith((l, r) => ByteUtil.UnsafeComparer.INSTANCE.compareTo(l, r) <= 0)
+ blockletList += sorted.head
+ blockletList += sorted.last
+ }
+
+
+ /**
+ * This is called during closing of writer.So after this call no more data will be sent to this
+ * class.
+ */
+ override def finish(): Unit = {
+ val out = new ByteOutputStream()
+ val outStream = new ObjectOutputStream(out)
+ outStream.writeObject(maxMin)
+ outStream.close()
+ val bytes = compressor.compressByte(out.getBytes)
+ stream.write(bytes)
+ stream.writeInt(bytes.length)
+ stream.close()
+ commitFile(cgwritepath)
+ }
+
+
+}
+
+class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
+
+ val file2 = resourcesPath + "/compaction/fil2.csv"
+ override protected def beforeAll(): Unit = {
+ //n should be about 5000000 of reset if size is default 1024
+ val n = 150000
+ CompactionSupportGlobalSortBigFileTest.createFile(file2, n * 4, n)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql(
+ """
+ | CREATE TABLE normal_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE normal_test OPTIONS('header'='false')")
+ }
+
+ test("test cg datamap") {
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ sql(
+ """
+ | CREATE TABLE datamap_test_cg(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test_cg")
+ // register datamap writer
+ sql(s"create datamap cgdatamap on table datamap_test_cg using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_cg OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test_cg where name='n502670'"),
+ sql("select * from normal_test where name='n502670'"))
+ }
+
+ test("test cg datamap with 2 datamaps ") {
+ sql("DROP TABLE IF EXISTS datamap_test")
+ sql(
+ """
+ | CREATE TABLE datamap_test(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val table = CarbonMetadata.getInstance().getCarbonTable("default_datamap_test")
+ // register datamap writer
+ sql(s"create datamap ggdatamap1 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='name')")
+ sql(s"create datamap ggdatamap2 on table datamap_test using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('indexcolumns'='city')")
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test OPTIONS('header'='false')")
+ checkAnswer(sql("select * from datamap_test where name='n502670' and city='c2670'"),
+ sql("select * from normal_test where name='n502670' and city='c2670'"))
+ }
+
+ override protected def afterAll(): Unit = {
+ CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
+ sql("DROP TABLE IF EXISTS normal_test")
+ sql("DROP TABLE IF EXISTS datamap_test_cg")
+ }
+}