You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:40:30 UTC
[05/50] [abbrv] carbondata git commit: [CARBONDATA-2489] Coverity
scan fixes
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index d3b9b48..2115f82 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -159,7 +159,8 @@ public class UnsafeMemoryManager {
/**
* It tries to allocate memory of `size` bytes, keep retry until it allocates successfully.
*/
- public static MemoryBlock allocateMemoryWithRetry(long taskId, long size) throws MemoryException {
+ public static MemoryBlock allocateMemoryWithRetry(long taskId, long size)
+ throws MemoryException {
MemoryBlock baseBlock = null;
int tries = 0;
while (tries < 300) {
@@ -177,8 +178,7 @@ public class UnsafeMemoryManager {
tries++;
}
if (baseBlock == null) {
- LOGGER.error(" Memory Used : " + INSTANCE.memoryUsed + " Tasks running : "
- + taskIdToMemoryBlockMap.keySet());
+ INSTANCE.printCurrentMemoryUsage();
throw new MemoryException("Not enough memory");
}
return baseBlock;
@@ -187,4 +187,9 @@ public class UnsafeMemoryManager {
public static boolean isOffHeap() {
return offHeap;
}
+
+ private synchronized void printCurrentMemoryUsage() {
+ LOGGER.error(
+ " Memory Used : " + memoryUsed + " Tasks running : " + taskIdToMemoryBlockMap.keySet());
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index 0f0f120..94a4e89 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -31,20 +31,32 @@ public class ArrayType extends DataType {
return true;
}
- public DataType getElementType() {
- return elementType;
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ArrayType)) {
+ return false;
+ }
+ if (!this.getName().equalsIgnoreCase(((ArrayType) obj).getName())) {
+ return false;
+ }
+ return true;
}
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof ArrayType)) return false;
-
- ArrayType arrayType = (ArrayType) o;
-
- return elementType.equals(arrayType.elementType);
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + getName().hashCode();
+ return result;
}
- @Override public int hashCode() {
- return elementType.hashCode();
+ public DataType getElementType() {
+ return elementType;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
index b2acd21..8536222 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalType.java
@@ -29,6 +29,37 @@ public class DecimalType extends DataType {
this.scale = scale;
}
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof DecimalType)) {
+ return false;
+ }
+ if (!this.getName().equalsIgnoreCase(((DecimalType) obj).getName())) {
+ return false;
+ }
+ if (this.precision != ((DecimalType) obj).precision) {
+ return false;
+ }
+ if (this.scale != ((DecimalType) obj).scale) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + getName().hashCode();
+ return result;
+ }
+
public int getPrecision() {
return precision;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
index 97cc4f0..90b7374 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
@@ -33,6 +33,31 @@ public class StructType extends DataType {
return true;
}
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof StructType)) {
+ return false;
+ }
+ if (!this.getName().equalsIgnoreCase(((StructType) obj).getName())) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + getName().hashCode();
+ return result;
+ }
+
public List<StructField> getFields() {
return fields;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 80c6a3a..fb4d8e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -33,15 +33,11 @@ import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.metadata.schema.table.WritableUtil;
import org.apache.carbondata.core.preagg.TimeSeriesUDF;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
/**
* Store the information about the column meta data present the table
*/
public class ColumnSchema implements Serializable, Writable {
- private static final Log LOG = LogFactory.getLog(ColumnSchema.class);
/**
* serialization version
*/
@@ -334,8 +330,6 @@ public class ColumnSchema implements Serializable, Writable {
return false;
}
} else if (!columnName.equals(other.columnName)) {
- LOG.error("column name is " + columnName
- + " but other column name is " + other.columnName);
return false;
}
if (dataType == null) {
@@ -343,8 +337,6 @@ public class ColumnSchema implements Serializable, Writable {
return false;
}
} else if (!dataType.equals(other.dataType)) {
- LOG.error("column name is" + columnName + " data type is " + dataType
- + " but other column data type is " + other.dataType);
return false;
}
return true;
@@ -361,40 +353,16 @@ public class ColumnSchema implements Serializable, Writable {
return false;
}
ColumnSchema other = (ColumnSchema) obj;
- if (!columnUniqueId.equals(other.columnUniqueId)) {
- LOG.error("Index file's column " + columnName + " columnUniqueId is " + columnUniqueId
- + " but table's column columnUniqueId is " + other.columnUniqueId);
- return false;
- }
- if (isDimensionColumn != other.isDimensionColumn) {
- LOG.error("Index file's column " + columnName + " isDimensionColumn is " + isDimensionColumn
- + " but table's column isDimensionColumn is " + other.isDimensionColumn);
- return false;
- }
- if (scale != other.scale) {
- LOG.error("Index file's column " + columnName + " scale is " + scale
- + " but table's column scale is " + other.scale);
- return false;
- }
- if (precision != other.precision) {
- LOG.error("Index file's column " + columnName + " precision is " + precision
- + " but table's column precision is " + other.precision);
- return false;
- }
- if (isSortColumn != other.isSortColumn) {
- LOG.error("Index file's column " + columnName + " isSortColumn is " + isSortColumn
- + " but table's column isSortColumn is " + other.isSortColumn);
+ if (!columnUniqueId.equals(other.columnUniqueId) ||
+ (isDimensionColumn != other.isDimensionColumn) ||
+ (isSortColumn != other.isSortColumn)) {
return false;
}
if (encodingList.size() != other.encodingList.size()) {
- LOG.error("Index file's column " + columnName + " encoding size is " + encodingList.size()
- + " but table's column encoding size is " + other.encodingList.size());
return false;
}
for (int i = 0; i < encodingList.size(); i++) {
if (encodingList.get(i).compareTo(other.encodingList.get(i)) != 0) {
- LOG.error("Index file's column " + columnName + " encoding is " + encodingList.get(i)
- + " but table's column encoding is " + other.encodingList.get(i));
return false;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
index df712de..ce0f15d 100644
--- a/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
+++ b/core/src/main/java/org/apache/carbondata/core/preagg/TimeSeriesUDF.java
@@ -112,7 +112,7 @@ public class TimeSeriesUDF {
/**
* Below method will be used to initialize the thread local
*/
- private synchronized void initialize() {
+ private void initialize() {
if (calanderThreadLocal.get() == null) {
calanderThreadLocal.set(new GregorianCalendar());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 6439b36..4f41b92 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -111,6 +111,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
// get the thrift object for dictionary chunk
dictionaryChunkMeta = (ColumnDictionaryChunkMeta) dictionaryMetadataFileReader.read();
}
+ if (null == dictionaryChunkMeta) {
+ throw new IOException("Last dictionary chunk does not exist");
+ }
// create a new instance of chunk meta wrapper using thrift object
return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
}
@@ -128,6 +131,9 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
break;
}
}
+ if (null == dictionaryChunkMeta) {
+ throw new IOException("Matching dictionary chunk does not exist");
+ }
// create a new instance of chunk meta wrapper using thrift object
return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
index 1c440cf..33f0db7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedRawResultCollector.java
@@ -197,7 +197,7 @@ public class RestructureBasedRawResultCollector extends RawBasedResultCollector
if (CarbonUtil.hasEncoding(actualQueryDimensions[i].getDimension().getEncoder(),
Encoding.DICTIONARY)) {
// if dimension exists then add the key array value else add the default value
- if (dimensionInfo.getDimensionExists()[i]) {
+ if (dimensionInfo.getDimensionExists()[i] && null != keyArray && 0 != keyArray.length) {
keyArrayWithNewAddedColumns[newKeyArrayIndex++] =
keyArray[existingColumnKeyArrayIndex++];
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
index 04669ab..aed472c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
@@ -70,9 +70,7 @@ public class SearchModeDetailQueryExecutor extends AbstractQueryExecutor<Object>
public CarbonIterator<Object> execute(QueryModel queryModel)
throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- if (executorService == null) {
- initThreadPool();
- }
+
this.queryIterator = new SearchModeResultIterator(
blockExecutionInfoList,
queryModel,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
index 6c9396b..00fd511 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
@@ -74,9 +74,7 @@ public class SearchModeVectorDetailQueryExecutor extends AbstractQueryExecutor<O
public CarbonIterator<Object> execute(QueryModel queryModel)
throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- if (executorService == null) {
- initThreadPool();
- }
+
this.queryIterator = new SearchModeVectorResultIterator(
blockExecutionInfoList,
queryModel,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index dc9415e..74c9ae2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -207,12 +207,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
*/
private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter,
List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode) {
-
+ if (null == dataRefNode.getColumnsMinValue() || null == dataRefNode.getColumnsMaxValue()) {
+ listOfDataBlocksToScan.add(dataRefNode);
+ return;
+ }
BitSet bitSet = filterExecuter
.isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue());
if (!bitSet.isEmpty()) {
listOfDataBlocksToScan.add(dataRefNode);
-
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 5196f8f..9741915 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -77,12 +77,10 @@ import org.apache.carbondata.core.scan.expression.logical.AndExpression;
import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
import org.apache.carbondata.core.scan.filter.executer.AndFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.DimColumnExecuterFilterInfo;
-import org.apache.carbondata.core.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.ExcludeFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.FalseFilterExecutor;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
-import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.MeasureColumnExecuterFilterInfo;
import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
@@ -232,30 +230,24 @@ public final class FilterUtil {
msrColResolvedFilterInfo, true);
}
}
- if (null != dimColResolvedFilterInfo) {
- CarbonDimension dimension = dimColResolvedFilterInfo.getDimension();
- if (dimension.hasEncoding(Encoding.IMPLICIT)) {
- return new ImplicitIncludeFilterExecutorImpl(dimColResolvedFilterInfo);
- } else if (dimension.isColumnar()) {
- CarbonDimension dimensionFromCurrentBlock =
- segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension());
- if (null != dimensionFromCurrentBlock) {
- // update dimension and column index according to the dimension position in current block
- DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
- dimColResolvedFilterInfo.getCopyObject();
- dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
- dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null,
- segmentProperties, false);
- } else {
- return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, false);
- }
+ CarbonDimension dimension = dimColResolvedFilterInfo.getDimension();
+ if (dimension.hasEncoding(Encoding.IMPLICIT)) {
+ return new ImplicitIncludeFilterExecutorImpl(dimColResolvedFilterInfo);
+ } else {
+ CarbonDimension dimensionFromCurrentBlock =
+ segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ // update dimension and column index according to the dimension position in current block
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+ dimColResolvedFilterInfo.getCopyObject();
+ dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+ dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null,
+ segmentProperties, false);
} else {
- return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
+ msrColResolvedFilterInfo, false);
}
- } else {
- return new IncludeColGroupFilterExecuterImpl(null, segmentProperties);
}
}
@@ -288,24 +280,19 @@ public final class FilterUtil {
msrColResolvedFilterInfo, true);
}
}
- if ((null != dimColResolvedFilterInfo) && (dimColResolvedFilterInfo.getDimension()
- .isColumnar())) {
- CarbonDimension dimensionFromCurrentBlock =
- segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension());
- if (null != dimensionFromCurrentBlock) {
- // update dimension and column index according to the dimension position in current block
- DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
- dimColResolvedFilterInfo.getCopyObject();
- dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
- dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null,
- segmentProperties, false);
- } else {
- return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
- msrColResolvedFilterInfo, false);
- }
+ CarbonDimension dimensionFromCurrentBlock =
+ segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ // update dimension and column index according to the dimension position in current block
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+ dimColResolvedFilterInfo.getCopyObject();
+ dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+ dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null,
+ segmentProperties, false);
} else {
- return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
+ msrColResolvedFilterInfo, false);
}
}
@@ -581,17 +568,15 @@ public final class FilterUtil {
}
Collections.sort(surrogates);
ColumnFilterInfo columnFilterInfo = null;
- if (surrogates.size() > 0) {
- columnFilterInfo = new ColumnFilterInfo();
- if (isExcludeFilterNeedsToApply) {
- columnFilterInfo.setOptimized(true);
- }
- columnFilterInfo.setIncludeFilter(isIncludeFilter);
- if (!isIncludeFilter) {
- columnFilterInfo.setExcludeFilterList(surrogates);
- } else {
- columnFilterInfo.setFilterList(surrogates);
- }
+ columnFilterInfo = new ColumnFilterInfo();
+ if (isExcludeFilterNeedsToApply) {
+ columnFilterInfo.setOptimized(true);
+ }
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ if (!isIncludeFilter) {
+ columnFilterInfo.setExcludeFilterList(surrogates);
+ } else {
+ columnFilterInfo.setFilterList(surrogates);
}
return columnFilterInfo;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
deleted file mode 100644
index 44f7c07..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.filter.executer;
-
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-/**
- * It checks if filter is required on given block and if required, it does
- * linear search on block data and set the bitset.
- */
-public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
-
- /**
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- */
- public ExcludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- super(dimColResolvedFilterInfo, null, segmentProperties, false);
- }
-
- /**
- * Check if scan is required on given block based on min and max value
- */
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- bitSet.flip(0, 1);
- return bitSet;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
deleted file mode 100644
index e4da26f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
-import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
-import org.apache.carbondata.core.util.BitSetGroup;
-import org.apache.carbondata.core.util.ByteUtil;
-
-/**
- * It checks if filter is required on given block and if required, it does
- * linear search on block data and set the bitset.
- */
-public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl {
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(IncludeColGroupFilterExecuterImpl.class.getName());
-
- /**
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- */
- public IncludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- super(dimColResolvedFilterInfo, null, segmentProperties, false);
- }
-
- /**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
-
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.set(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- @Override
- public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
- boolean useBitsetPipeLine) throws IOException {
- int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
- .get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
- rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
- rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
- rawBlockletColumnChunks.getFileReader(), chunkIndex);
- }
- DimensionRawColumnChunk dimensionRawColumnChunk =
- rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
- BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
- for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
- if (dimensionRawColumnChunk.getMaxValues() != null) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
- dimensionRawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
- }
- }
- return bitSetGroup;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
- * Check if scan is required on given block based on min and max value
- */
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
- int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex);
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue);
- // and filter-min should be positive
- int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue);
-
- // if any filter value is in range than this block needs to be
- // scanned
- if (maxCompare <= 0 && minCompare >= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
- }
-
- /**
- * It extract min and max data for given column from stored min max value
- *
- * @param colGrpColumns
- * @param minMaxData
- * @param columnIndex
- * @return
- */
- private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) {
- int startIndex = 0;
- int endIndex = 0;
- if (null != colGrpColumns) {
- for (int i = 0; i < colGrpColumns.length; i++) {
- int colGrpId = getColumnGroupId(colGrpColumns[i]);
- int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId);
- int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal);
- int colSize = 0;
- for (int j = byteRange[0]; j <= byteRange[1]; j++) {
- colSize++;
- }
- if (colGrpColumns[i] == columnIndex) {
- endIndex = startIndex + colSize;
- break;
- }
- startIndex += colSize;
- }
- }
- byte[] data = new byte[endIndex - startIndex];
- System.arraycopy(minMaxData, startIndex, data, 0, data.length);
- return data;
- }
-
- /**
- * It returns column groups which have provided column ordinal
- *
- * @param columnIndex
- * @return column group array
- */
- private int[] getAllColumns(int columnIndex) {
- int[][] colGroups = segmentProperties.getColumnGroups();
- for (int i = 0; i < colGroups.length; i++) {
- if (QueryUtil.searchInArray(colGroups[i], columnIndex)) {
- return colGroups[i];
- }
- }
- return null;
- }
-
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 34555e1..b218813 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -113,7 +113,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
*/
private void ifDefaultValueMatchesFilter() {
isDefaultValuePresentInFilter = false;
- if (!this.isDimensionPresentInCurrentBlock) {
+ if (!this.isDimensionPresentInCurrentBlock && null != filterRangesValues) {
CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
index e1432b0..f901238 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
@@ -134,5 +134,4 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
}
return isDefaultValuePresentInFilterValues;
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 4bee89b..057a244 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -124,8 +124,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
}
if (this.dimColEvaluatorInfoList.size() > 0) {
- this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
- this.dimensionChunkIndex = new int[dimColEvaluatorInfoList.size()];
+ this.isDimensionPresentInCurrentBlock = new boolean[this.dimColEvaluatorInfoList.size()];
+ this.dimensionChunkIndex = new int[this.dimColEvaluatorInfoList.size()];
} else {
this.isDimensionPresentInCurrentBlock = new boolean[]{false};
this.dimensionChunkIndex = new int[]{0};
@@ -136,8 +136,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
this.msrColEvalutorInfoList = msrColEvalutorInfoList;
}
if (this.msrColEvalutorInfoList.size() > 0) {
- this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
- this.measureChunkIndex = new int[msrColEvalutorInfoList.size()];
+ this.isMeasurePresentInCurrentBlock = new boolean[this.msrColEvalutorInfoList.size()];
+ this.measureChunkIndex = new int[this.msrColEvalutorInfoList.size()];
} else {
this.isMeasurePresentInCurrentBlock = new boolean[]{false};
this.measureChunkIndex = new int[] {0};
@@ -647,13 +647,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
}
- if (null != msrColEvalutorInfoList) {
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) {
- rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] =
- rawBlockletColumnChunks.getDataBlock()
- .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]);
- }
+ for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), measureChunkIndex[0]);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index be871d4..e339a58 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -67,7 +67,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
this.filterRangeValues = filterRangeValues;
this.msrFilterRangeValues = msrFilterRangeValues;
lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
- if (!msrColEvalutorInfoList.isEmpty()) {
+ if (!this.msrColEvalutorInfoList.isEmpty()) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
comparator = Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
}
@@ -98,9 +98,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
} else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
byte[] defaultValue = measure.getDefaultValue();
+ SerializableComparator comparatorTmp = (null != comparator ?
+ comparator :
+ Comparator.getComparatorByDataTypeForMeasure(measure.getDataType()));
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
- int maxCompare = comparator.compare(msrFilterRangeValues[k],
+ int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k],
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index a3359be..2ea3c73 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -98,9 +98,11 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
} else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
byte[] defaultValue = measure.getDefaultValue();
+ SerializableComparator comparatorTmp =
+ Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
- int maxCompare = comparator.compare(msrFilterRangeValues[k],
+ int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k],
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));
if (maxCompare <= 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 0c268c9..f52d087 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -100,9 +100,11 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
} else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
byte[] defaultValue = measure.getDefaultValue();
+ SerializableComparator comparatorTmp =
+ Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
- int maxCompare = comparator.compare(msrFilterRangeValues[k],
+ int maxCompare = comparatorTmp.compare(msrFilterRangeValues[k],
RestructureUtil.getMeasureDefaultValue(measure.getColumnSchema(),
measure.getDefaultValue()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
index c5ed77d..97e750a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -100,12 +100,14 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
} else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
byte[] defaultValue = measure.getDefaultValue();
+ SerializableComparator comparatorTmp =
+ Comparator.getComparatorByDataTypeForMeasure(measure.getDataType());
if (null != defaultValue) {
for (int k = 0; k < msrFilterRangeValues.length; k++) {
Object convertedValue = RestructureUtil
.getMeasureDefaultValue(measure.getColumnSchema(), measure.getDefaultValue());
int maxCompare =
- comparator.compare(msrFilterRangeValues[k], convertedValue);
+ comparatorTmp.compare(msrFilterRangeValues[k], convertedValue);
if (maxCompare > 0) {
isDefaultValuePresentInFilter = true;
break;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index ccbbc32..8ad0c48 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -303,7 +303,6 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
this.dimColResolvedFilterInfo.getDimension(), segmentProperties, false);
}
return null;
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index d975c20..01aa939 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -135,14 +135,19 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
} else {
DataRefNode startDataBlock =
finder.findFirstDataBlock(dataRefNode, blockInfo.getStartKey());
- while (startDataBlock.nodeIndex() < blockInfo.getStartBlockletIndex()) {
+ while ((null != startDataBlock) && (startDataBlock.nodeIndex() < blockInfo
+ .getStartBlockletIndex())) {
startDataBlock = startDataBlock.getNextDataRefNode();
}
long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
//if number of block is less than 0 then take end block.
if (numberOfBlockToScan <= 0) {
DataRefNode endDataBlock = finder.findLastDataBlock(dataRefNode, blockInfo.getEndKey());
- numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1;
+ if (null != startDataBlock) {
+ numberOfBlockToScan = endDataBlock.nodeIndex() - startDataBlock.nodeIndex() + 1;
+ } else {
+ numberOfBlockToScan = endDataBlock.nodeIndex() + 1;
+ }
}
blockInfo.setFirstDataBlock(startDataBlock);
blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
index 57d8177..0100c8b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -110,6 +110,10 @@ public class BlockletFilterScanner extends BlockletFullScanner {
totalPagesScanned.getCount() + dataBlock.numberOfPages());
// apply min max
if (isMinMaxEnabled) {
+ if (null == dataBlock.getColumnsMaxValue()
+ || null == dataBlock.getColumnsMinValue()) {
+ return true;
+ }
BitSet bitSet = null;
// check for implicit include filter instance
if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index d6671b4..9dc8fe6 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -230,7 +230,13 @@ public class LoadMetadataDetails implements Serializable {
* @return
*/
public long getLoadStartTimeAsLong() {
- return (!loadStartTime.isEmpty()) ? getTimeStamp(loadStartTime) : 0;
+ if (!loadStartTime.isEmpty()) {
+ Long time = getTimeStamp(loadStartTime);
+ if (null != time) {
+ return time;
+ }
+ }
+ return 0;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 363b5bc..1c53fbb 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -322,13 +322,12 @@ public class SegmentUpdateStatusManager {
* @return the list of delete file
*/
private List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple,
- final String extension,
- String segment) {
+ final String extension, String segment) throws IOException {
List<String> deleteFileList = new ArrayList<>();
for (SegmentUpdateDetails block : updateDetails) {
- if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) &&
- block.getSegmentName().equalsIgnoreCase(segment) &&
- !CarbonUpdateUtil.isBlockInvalid(block.getSegmentStatus())) {
+ if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && block.getSegmentName()
+ .equalsIgnoreCase(segment) && !CarbonUpdateUtil
+ .isBlockInvalid(block.getSegmentStatus())) {
final long deltaStartTimestamp = getStartTimeOfDeltaFile(extension, block);
// If there is no delete delete file , then return null
if (deltaStartTimestamp == 0) {
@@ -347,33 +346,38 @@ public class SegmentUpdateStatusManager {
private List<String> getFilePaths(CarbonFile blockDir, final String blockNameFromTuple,
final String extension, List<String> deleteFileList, final long deltaStartTimestamp,
- final long deltaEndTimeStamp) {
- CarbonFile[] files = blockDir.getParentFile().listFiles(new CarbonFileFilter() {
-
- @Override public boolean accept(CarbonFile pathName) {
- String fileName = pathName.getName();
- if (fileName.endsWith(extension) && pathName.getSize() > 0) {
- String firstPart = fileName.substring(0, fileName.indexOf('.'));
- String blockName =
- firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
- long timestamp = Long.parseLong(firstPart
- .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
- firstPart.length()));
- if (blockNameFromTuple.equals(blockName) && (
- (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (
- Long.compare(timestamp, deltaStartTimestamp) >= 0))) {
- return true;
+ final long deltaEndTimeStamp) throws IOException {
+ if (null != blockDir.getParentFile()) {
+ CarbonFile[] files = blockDir.getParentFile().listFiles(new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile pathName) {
+ String fileName = pathName.getName();
+ if (fileName.endsWith(extension) && pathName.getSize() > 0) {
+ String firstPart = fileName.substring(0, fileName.indexOf('.'));
+ String blockName =
+ firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
+ long timestamp = Long.parseLong(firstPart
+ .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
+ firstPart.length()));
+ if (blockNameFromTuple.equals(blockName) && (
+ (Long.compare(timestamp, deltaEndTimeStamp) <= 0) && (
+ Long.compare(timestamp, deltaStartTimestamp) >= 0))) {
+ return true;
+ }
}
+ return false;
}
- return false;
- }
- });
+ });
- for (CarbonFile cfile : files) {
- if (null == deleteFileList) {
- deleteFileList = new ArrayList<String>(files.length);
+ for (CarbonFile cfile : files) {
+ if (null == deleteFileList) {
+ deleteFileList = new ArrayList<String>(files.length);
+ }
+ deleteFileList.add(cfile.getCanonicalPath());
}
- deleteFileList.add(cfile.getCanonicalPath());
+ } else {
+ throw new IOException("Parent file could not found");
}
return deleteFileList;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index b74c279..e30ad03 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -60,7 +60,12 @@ public abstract class AbstractDataFileFooterConverter {
*/
private static BitSet getPresenceMeta(
org.apache.carbondata.format.PresenceMeta presentMetadataThrift) {
- return BitSet.valueOf(presentMetadataThrift.getPresent_bit_stream());
+ final byte[] present_bit_stream = presentMetadataThrift.getPresent_bit_stream();
+ if (null != present_bit_stream) {
+ return BitSet.valueOf(present_bit_stream);
+ } else {
+ return new BitSet(1);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 9880b4d..af5121c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -370,7 +370,7 @@ public class CarbonMetadataUtil {
} else if (DataTypes.isDecimal(dataType)) {
return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));
} else {
- throw new IllegalArgumentException("Invalid data type");
+ throw new IllegalArgumentException("Invalid data type:" + dataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 9822167..f7f71b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -587,20 +587,16 @@ public final class DataTypeUtil {
return null;
}
try {
- Object parsedValue = null;
if (actualDataType == DataTypes.SHORT) {
- parsedValue = Short.parseShort(data);
+ Short.parseShort(data);
} else if (actualDataType == DataTypes.INT) {
- parsedValue = Integer.parseInt(data);
+ Integer.parseInt(data);
} else if (actualDataType == DataTypes.LONG) {
- parsedValue = Long.parseLong(data);
+ Long.parseLong(data);
} else {
return data;
}
- if (null != parsedValue) {
- return data;
- }
- return null;
+ return data;
} catch (NumberFormatException ex) {
return null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 62192ff..e8a121c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -576,11 +576,15 @@ public class CarbonTablePath {
*/
public static CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir,
final String columnUniqueId) {
- return sortIndexDir.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
- }
- });
+ if (null != sortIndexDir) {
+ return sortIndexDir.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().startsWith(columnUniqueId) && file.getName()
+ .endsWith(SORT_INDEX_EXT);
+ }
+ });
+ }
+ return null;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
deleted file mode 100644
index 54b66a6..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.impl;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class ColumnGroupDimensionDataChunkTest {
-
- static ColumnGroupDimensionColumnPage columnGroupDimensionDataChunk;
- static KeyGenerator keyGenerator;
-
- @BeforeClass public static void setup() {
- int[] bitLength = CarbonUtil.getDimensionBitLength(new int[] { 10, 10, 10 }, new int[] { 3 });
- // create a key generator
- keyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
- byte[] data = new byte[keyGenerator.getKeySizeInBytes() * 3];
- int position = 0;
- for (int i = 1; i <= 3; i++) {
- try {
- System.arraycopy(keyGenerator.generateKey(new int[] { i, i, i }), 0, data, position,
- keyGenerator.getKeySizeInBytes());
- } catch (KeyGenException e) {
- assertTrue(false);
- }
- position += keyGenerator.getKeySizeInBytes();
- }
- columnGroupDimensionDataChunk =
- new ColumnGroupDimensionColumnPage(data, keyGenerator.getKeySizeInBytes(), 3);
- }
-
- @Test public void fillChunkDataTest() {
- List<Integer> ordinals = new ArrayList<Integer>();
- ordinals.add(1);
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
- byte[] buffer = new byte[1];
- columnGroupDimensionDataChunk.fillRawData(1, 0, buffer, keyStructureInfo);
- assertEquals(buffer[0], 2);
- }
-
- @Test public void getChunkDataTest() {
- byte[] b = { 34, 2 };
- byte res[] = columnGroupDimensionDataChunk.getChunkData(1);
- Assert.assertTrue(Arrays.equals(res, b));
- }
-
- @Test public void fillConvertedChunkDataTest() {
- int[] row = new int[3];
- int[] expected = { 0, 0, 3 };
- List<Integer> ordinals = new ArrayList<Integer>();
- ordinals.add(2);
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo(ordinals, keyGenerator);
- keyStructureInfo.setMdkeyQueryDimensionOrdinal(new int[] { 2 });
- int res = columnGroupDimensionDataChunk.fillSurrogateKey(2, 2, row, keyStructureInfo);
- Assert.assertTrue(Arrays.equals(row, expected));
- }
-
- /**
- * Below method will be used to get the key structure info for the query
- *
- * @param ordinals query model
- * @param keyGenerator
- * @return key structure info
- */
- private KeyStructureInfo getKeyStructureInfo(List<Integer> ordinals, KeyGenerator keyGenerator) {
- // getting the masked byte range for dictionary column
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(ordinals, keyGenerator);
-
- // getting the masked bytes for query dimension dictionary column
- int[] maskedBytes = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
-
- // max key for the dictionary dimension present in the query
- byte[] maxKey = null;
- try {
- // getting the max key which will be used to masked and get the
- // masked key
- maxKey = QueryUtil.getMaxKeyBasedOnOrinal(ordinals, keyGenerator);
- } catch (KeyGenException e) {
- }
-
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
index f55cd67..890d36d 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
@@ -59,8 +59,11 @@ public class AlluxioCarbonFileTest {
}
try {
FileOutputStream oFile = new FileOutputStream(file, true);
+ oFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
}
fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
index ba661b1..82d1501 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -54,14 +54,17 @@ public class ViewFsCarbonFileTest {
file = new File("Test.carbondata");
if (!file.exists())
try {
- file.createNewFile();
+ file.createNewFile();
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace();
}
try {
- FileOutputStream oFile = new FileOutputStream(file, true);
+ FileOutputStream oFile = new FileOutputStream(file, true);
+ oFile.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
}
fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
index 32af8d3..8be1e2e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CacheClient.java
@@ -16,39 +16,21 @@
*/
package org.apache.carbondata.hadoop;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
/**
* CacheClient : Holds all the Cache access clients for Btree, Dictionary
*/
public class CacheClient {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CacheClient.class.getName());
-
- private final Object lock = new Object();
-
// segment access client for driver LRU cache
private CacheAccessClient<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
segmentAccessClient;
- private static Map<SegmentTaskIndexStore.SegmentPropertiesWrapper, SegmentProperties>
- segmentProperties = new ConcurrentHashMap<>();
-
public CacheClient() {
Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper> segmentCache =
CacheProvider.getInstance().createCache(CacheType.DRIVER_BTREE);
@@ -63,35 +45,4 @@ public class CacheClient {
public void close() {
segmentAccessClient.close();
}
-
- /**
- * Method to get the segment properties and avoid construction of new segment properties until
- * the schema is not modified
- *
- * @param tableIdentifier
- * @param columnsInTable
- * @param columnCardinality
- */
- public SegmentProperties getSegmentProperties(AbsoluteTableIdentifier tableIdentifier,
- List<ColumnSchema> columnsInTable, int[] columnCardinality) {
- SegmentTaskIndexStore.SegmentPropertiesWrapper segmentPropertiesWrapper =
- new SegmentTaskIndexStore.SegmentPropertiesWrapper(tableIdentifier, columnsInTable,
- columnCardinality);
- SegmentProperties segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
- if (null == segmentProperties) {
- synchronized (lock) {
- segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
- if (null == segmentProperties) {
- // create a metadata details
- // this will be useful in query handling
- // all the data file metadata will have common segment properties we
- // can use first one to get create the segment properties
- LOGGER.info("Constructing new SegmentProperties");
- segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
- this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
- }
- }
- }
- return segmentProperties;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
index 0bcb188..e5e3165 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonOutputCommitter.java
@@ -124,6 +124,9 @@ public class CarbonOutputCommitter extends FileOutputCommitter {
.mergeSegmentFiles(readPath, segmentFileName,
CarbonTablePath.getSegmentFilesLocation(loadModel.getTablePath()));
if (segmentFile != null) {
+ if (null == newMetaEntry) {
+ throw new RuntimeException("Internal Error");
+ }
// Move all files from temp directory of each segment to partition directory
SegmentFileStore.moveFromTempFolder(segmentFile,
loadModel.getSegmentId() + "_" + loadModel.getFactTimeStamp() + ".tmp",
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
index 7d9c712..b619158 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
@@ -18,14 +18,10 @@
package org.apache.carbondata.hadoop.internal.segment;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
@@ -56,25 +52,6 @@ public abstract class Segment {
}
/**
- * return all InputSplit of this segment, each file is a InputSplit
- * @param job job context
- * @return all InputSplit
- * @throws IOException
- */
- public List<InputSplit> getAllSplits(JobContext job) throws IOException {
- List<InputSplit> result = new ArrayList<>();
- Path p = new Path(path);
- FileSystem fs = p.getFileSystem(job.getConfiguration());
-
- //TODO: filter out the hidden files
- FileStatus[] files = fs.globStatus(p);
- for (FileStatus file: files) {
- // make split and add to result
- }
- return result;
- }
-
- /**
* get all files, implementation may use the input filter and index to prune files
* @param job job context
* @param filterResolver filter
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index febca50..e95382c 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -196,12 +196,15 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T
if (carbonColumn.isDimension() && carbonColumn.getColumnSchema().getNumberOfChild() > 0) {
childCarbonDimensions = ((CarbonDimension) carbonColumn).getListOfChildDimensions();
}
- Writable[] arr = new Writable[objArray.length];
- for (int i = 0; i < objArray.length; i++) {
- arr[i] = createWritableObject(objArray[i], childCarbonDimensions.get(i));
+ if (null != childCarbonDimensions) {
+ Writable[] arr = new Writable[objArray.length];
+ for (int i = 0; i < objArray.length; i++) {
+
+ arr[i] = createWritableObject(objArray[i], childCarbonDimensions.get(i));
+ }
+ return new ArrayWritable(Writable.class, arr);
}
- return new ArrayWritable(Writable.class, arr);
}
throw new IOException("DataType not supported in Carbondata");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
index 89a5ed6..d4cf480 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
@@ -77,13 +77,17 @@ public class MapredCarbonInputFormat extends CarbonTableInputFormat<ArrayWritabl
}
}
}
- AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier
- .from(validInputPath, getDatabaseName(configuration), getTableName(configuration));
- // read the schema file to get the absoluteTableIdentifier having the correct table id
- // persisted in the schema
- CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
- configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable));
- setTableInfo(configuration, carbonTable.getTableInfo());
+ if (null != validInputPath) {
+ AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier
+ .from(validInputPath, getDatabaseName(configuration), getTableName(configuration));
+ // read the schema file to get the absoluteTableIdentifier having the correct table id
+ // persisted in the schema
+ CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
+ configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable));
+ setTableInfo(configuration, carbonTable.getTableInfo());
+ } else {
+ throw new InvalidPathException("No input paths specified in job");
+ }
}
private static CarbonTable getCarbonTable(Configuration configuration, String path)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
index 811393f..3a54b22 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataSplitManager.java
@@ -67,20 +67,23 @@ public class CarbondataSplitManager implements ConnectorSplitManager {
getColumnConstraints(layoutHandle.getConstraint());
CarbonTableCacheModel cache = carbonTableReader.getCarbonCache(key);
- Expression filters = PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint());
- try {
- List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters, layoutHandle.getConstraint());
-
- ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
- for (CarbonLocalInputSplit split : splits) {
- cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(),
- layoutHandle.getConstraint(), split, rebuildConstraints));
+ if (null != cache) {
+ Expression filters = PrestoFilterUtil.parseFilterExpression(layoutHandle.getConstraint());
+ try {
+ List<CarbonLocalInputSplit> splits = carbonTableReader.getInputSplits2(cache, filters,
+ layoutHandle.getConstraint());
+
+ ImmutableList.Builder<ConnectorSplit> cSplits = ImmutableList.builder();
+ for (CarbonLocalInputSplit split : splits) {
+ cSplits.add(new CarbondataSplit(connectorId, tableHandle.getSchemaTableName(),
+ layoutHandle.getConstraint(), split, rebuildConstraints));
+ }
+ return new FixedSplitSource(cSplits.build());
+ } catch (Exception ex) {
+ throw new RuntimeException(ex.getMessage(), ex);
}
- return new FixedSplitSource(cSplits.build());
- } catch (Exception ex) {
- throw new RuntimeException(ex.getMessage(), ex);
}
-
+ return null;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
index 3c42d0a..2c6a810 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
@@ -115,15 +115,21 @@ public class CarbonLocalInputSplit {
}
- public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) {
+ public static CarbonInputSplit convertSplit(CarbonLocalInputSplit carbonLocalInputSplit) {
CarbonInputSplit inputSplit = new CarbonInputSplit(carbonLocalInputSplit.getSegmentId(), "0",
new Path(carbonLocalInputSplit.getPath()), carbonLocalInputSplit.getStart(),
carbonLocalInputSplit.getLength(), carbonLocalInputSplit.getLocations()
.toArray(new String[carbonLocalInputSplit.getLocations().size()]),
- carbonLocalInputSplit.getNumberOfBlocklets(), ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()),
+ carbonLocalInputSplit.getNumberOfBlocklets(),
+ ColumnarFormatVersion.valueOf(carbonLocalInputSplit.getVersion()),
carbonLocalInputSplit.getDeleteDeltaFiles());
Gson gson = new Gson();
- BlockletDetailInfo blockletDetailInfo = gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class);
+ BlockletDetailInfo blockletDetailInfo =
+ gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class);
+
+ if (null == blockletDetailInfo) {
+ throw new RuntimeException("Could not read blocklet details");
+ }
try {
blockletDetailInfo.readColumnSchema(blockletDetailInfo.getColumnSchemaBinary());
} catch (IOException e) {
@@ -132,6 +138,4 @@ public class CarbonLocalInputSplit {
inputSplit.setDetailInfo(blockletDetailInfo);
return inputSplit;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
index 4507425..0b7206b 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/BooleanStreamReader.java
@@ -62,10 +62,8 @@ public class BooleanStreamReader extends AbstractStreamReader {
} else {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
- if (streamData != null) {
- for (int i = 0; i < numberOfRows; i++) {
- type.writeBoolean(builder, byteToBoolean(streamData[i]));
- }
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeBoolean(builder, byteToBoolean(streamData[i]));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ef91645/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
index 23db769..3e7fc59 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/readers/DoubleStreamReader.java
@@ -59,7 +59,7 @@ public class DoubleStreamReader extends AbstractStreamReader {
numberOfRows = batchSize;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
if (columnVector != null) {
- if(isDictionary) {
+ if (isDictionary) {
populateDictionaryVector(type, numberOfRows, builder);
} else {
if (columnVector.anyNullsSet()) {
@@ -72,10 +72,8 @@ public class DoubleStreamReader extends AbstractStreamReader {
} else {
numberOfRows = streamData.length;
builder = type.createBlockBuilder(new BlockBuilderStatus(), numberOfRows);
- if (streamData != null) {
- for (int i = 0; i < numberOfRows; i++) {
- type.writeDouble(builder, (Double) streamData[i]);
- }
+ for (int i = 0; i < numberOfRows; i++) {
+ type.writeDouble(builder, (Double) streamData[i]);
}
}