You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/03/16 09:28:48 UTC
[05/14] incubator-carbondata git commit: Following changes are done
as part of this commit.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
deleted file mode 100644
index b992ea6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.executor.infos;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * Info class which store all the details
- * which is required during aggregation
- */
-public class AggregatorInfo {
-
- /**
- * selected query measure ordinal
- * which will be used to read the measures chunk data
- * this will be storing the index of the measure in measures chunk
- */
- private int[] measureOrdinals;
-
- /**
- * This parameter will be used to
- * check whether particular measure is present
- * in the table block, if not then its default value will be used
- */
- private boolean[] measureExists;
-
- /**
- * this default value will be used to when some measure is not present
- * in the table block, in case of restructuring of the table if user is adding any
- * measure then in older block that measure wont be present so for measure default value
- * will be used to aggregate in the older table block query execution
- */
- private Object[] defaultValues;
-
- /**
- * Datatype of each measure;
- */
- private DataType[] measureDataTypes;
-
- /**
- * @return the measureOrdinal
- */
- public int[] getMeasureOrdinals() {
- return measureOrdinals;
- }
-
- /**
- * @param measureOrdinal the measureOrdinal to set
- */
- public void setMeasureOrdinals(int[] measureOrdinal) {
- this.measureOrdinals = measureOrdinal;
- }
-
- /**
- * @return the measureExists
- */
- public boolean[] getMeasureExists() {
- return measureExists;
- }
-
- /**
- * @param measureExists the measureExists to set
- */
- public void setMeasureExists(boolean[] measureExists) {
- this.measureExists = measureExists;
- }
-
- /**
- * @return the defaultValues
- */
- public Object[] getDefaultValues() {
- return defaultValues;
- }
-
- /**
- * @param defaultValues the defaultValues to set
- */
- public void setDefaultValues(Object[] defaultValues) {
- this.defaultValues = defaultValues;
- }
-
- public DataType[] getMeasureDataTypes() {
- return measureDataTypes;
- }
-
- public void setMeasureDataTypes(DataType[] measureDataTypes) {
- this.measureDataTypes = measureDataTypes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 2dd6721..b294b58 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -49,10 +49,10 @@ public class BlockExecutionInfo {
private boolean isFixedKeyUpdateRequired;
/**
- * below to store all the information required for aggregation during query
+ * below to store all the information required for measures during query
* execution
*/
- private AggregatorInfo aggregatorInfo;
+ private MeasureInfo measureInfo;
/**
* this will be used to get the first tentative block from which query
@@ -71,13 +71,6 @@ public class BlockExecutionInfo {
private String blockId;
/**
- * masked byte for block which will be used to unpack the fixed length key,
- * this will be used for updating the older block key with new block key
- * generator
- */
- private int[] maskedByteForBlock;
-
- /**
* total number of dimension in block
*/
private int totalNumberDimensionBlock;
@@ -196,21 +189,43 @@ public class BlockExecutionInfo {
}
/**
- * list of dimension selected for in query
+ * list of dimension present in the current block. This will be
+ * different in case of restructured block
*/
private QueryDimension[] queryDimensions;
/**
- * list of measure selected in query
+ * list of dimension selected for in query
+ */
+ private QueryDimension[] actualQueryDimensions;
+
+ /**
+ * list of dimension present in the current block. This will be
+ * different in case of restructured block
*/
private QueryMeasure[] queryMeasures;
/**
+ * list of measure selected in query
+ */
+ private QueryMeasure[] actualQueryMeasures;
+
+ /**
+ * variable to maintain dimension existence and default value info
+ */
+ private DimensionInfo dimensionInfo;
+
+ /**
* whether it needs to read data in vector/columnar format.
*/
private boolean vectorBatchCollector;
/**
+ * flag to specify that whether the current block is with latest schema or old schema
+ */
+ private boolean isRestructuredBlock;
+
+ /**
* absolute table identifier
*/
private AbsoluteTableIdentifier absoluteTableIdentifier;
@@ -247,15 +262,15 @@ public class BlockExecutionInfo {
/**
* @return the aggregatorInfos
*/
- public AggregatorInfo getAggregatorInfo() {
- return aggregatorInfo;
+ public MeasureInfo getMeasureInfo() {
+ return measureInfo;
}
/**
- * @param aggregatorInfo the aggregatorInfos to set
+ * @param measureInfo the aggregatorInfos to set
*/
- public void setAggregatorInfo(AggregatorInfo aggregatorInfo) {
- this.aggregatorInfo = aggregatorInfo;
+ public void setMeasureInfo(MeasureInfo measureInfo) {
+ this.measureInfo = measureInfo;
}
/**
@@ -287,22 +302,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the maskedByteForBlock
- */
- public int[] getMaskedByteForBlock() {
- return maskedByteForBlock;
- }
-
-
-
- /**
- * @param maskedByteForBlock the maskedByteForBlock to set
- */
- public void setMaskedByteForBlock(int[] maskedByteForBlock) {
- this.maskedByteForBlock = maskedByteForBlock;
- }
-
- /**
* @return the totalNumberDimensionBlock
*/
public int getTotalNumberDimensionBlock() {
@@ -359,20 +358,6 @@ public class BlockExecutionInfo {
}
/**
- * @return the restructureInfos
- */
- public KeyStructureInfo getKeyStructureInfo() {
- return keyStructureInfo;
- }
-
- /**
- * @param keyStructureInfo the restructureInfos to set
- */
- public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) {
- this.keyStructureInfo = keyStructureInfo;
- }
-
- /**
* @return the firstDataBlock
*/
public DataRefNode getFirstDataBlock() {
@@ -610,6 +595,38 @@ public class BlockExecutionInfo {
this.blockId = blockId;
}
+ public boolean isRestructuredBlock() {
+ return isRestructuredBlock;
+ }
+
+ public void setRestructuredBlock(boolean restructuredBlock) {
+ isRestructuredBlock = restructuredBlock;
+ }
+
+ public DimensionInfo getDimensionInfo() {
+ return dimensionInfo;
+ }
+
+ public void setDimensionInfo(DimensionInfo dimensionInfo) {
+ this.dimensionInfo = dimensionInfo;
+ }
+
+ public QueryDimension[] getActualQueryDimensions() {
+ return actualQueryDimensions;
+ }
+
+ public void setActualQueryDimensions(QueryDimension[] actualQueryDimensions) {
+ this.actualQueryDimensions = actualQueryDimensions;
+ }
+
+ public QueryMeasure[] getActualQueryMeasures() {
+ return actualQueryMeasures;
+ }
+
+ public void setActualQueryMeasures(QueryMeasure[] actualQueryMeasures) {
+ this.actualQueryMeasures = actualQueryMeasures;
+ }
+
public int[] getProjectionListDimensionIndexes() {
return projectionListDimensionIndexes;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
new file mode 100644
index 0000000..3b2e2bd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.executor.infos;
+
+/**
+ * This method will information about the query dimensions whether they exist in particular block
+ * and their default value
+ */
+public class DimensionInfo {
+
+ /**
+ * flag to check whether a given dimension exists in a given block
+ */
+ private boolean[] dimensionExists;
+
+ /**
+ * maintains default value for each dimension
+ */
+ private Object[] defaultValues;
+
+ /**
+ * @param dimensionExists
+ * @param defaultValues
+ */
+ public DimensionInfo(boolean[] dimensionExists, Object[] defaultValues) {
+ this.dimensionExists = dimensionExists;
+ this.defaultValues = defaultValues;
+ }
+
+ /**
+ * @return
+ */
+ public boolean[] getDimensionExists() {
+ return dimensionExists;
+ }
+
+ /**
+ * @return
+ */
+ public Object[] getDefaultValues() {
+ return defaultValues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
new file mode 100644
index 0000000..5dd4fb8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.executor.infos;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * Info class which store all the details
+ * which is required during aggregation
+ */
+public class MeasureInfo {
+
+ /**
+ * selected query measure ordinal
+ * which will be used to read the measures chunk data
+ * this will be storing the index of the measure in measures chunk
+ */
+ private int[] measureOrdinals;
+
+ /**
+ * This parameter will be used to
+ * check whether particular measure is present
+ * in the table block, if not then its default value will be used
+ */
+ private boolean[] measureExists;
+
+ /**
+ * this default value will be used to when some measure is not present
+ * in the table block, in case of restructuring of the table if user is adding any
+ * measure then in older block that measure wont be present so for measure default value
+ * will be used to aggregate in the older table block query execution
+ */
+ private Object[] defaultValues;
+
+ /**
+ * Datatype of each measure;
+ */
+ private DataType[] measureDataTypes;
+
+ /**
+ * @return the measureOrdinal
+ */
+ public int[] getMeasureOrdinals() {
+ return measureOrdinals;
+ }
+
+ /**
+ * @param measureOrdinal the measureOrdinal to set
+ */
+ public void setMeasureOrdinals(int[] measureOrdinal) {
+ this.measureOrdinals = measureOrdinal;
+ }
+
+ /**
+ * @return the measureExists
+ */
+ public boolean[] getMeasureExists() {
+ return measureExists;
+ }
+
+ /**
+ * @param measureExists the measureExists to set
+ */
+ public void setMeasureExists(boolean[] measureExists) {
+ this.measureExists = measureExists;
+ }
+
+ /**
+ * @return the defaultValues
+ */
+ public Object[] getDefaultValues() {
+ return defaultValues;
+ }
+
+ /**
+ * @param defaultValues the defaultValues to set
+ */
+ public void setDefaultValues(Object[] defaultValues) {
+ this.defaultValues = defaultValues;
+ }
+
+ public DataType[] getMeasureDataTypes() {
+ return measureDataTypes;
+ }
+
+ public void setMeasureDataTypes(DataType[] measureDataTypes) {
+ this.measureDataTypes = measureDataTypes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index ef6fb8a..2f47b6a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -208,6 +208,7 @@ public class QueryUtil {
*/
public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+ Map<Integer, Integer> queryDimensionToCurrentBlockDimensionOrdinalMapping,
List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
Set<Integer> allProjectionListDimensionIndexes) {
// using set as in row group columns will point to same block
@@ -219,16 +220,17 @@ public class QueryUtil {
continue;
}
- allProjectionListDimensionIndexes.add(
- dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal()));
+ Integer dimensionOrdinal = queryDimensionToCurrentBlockDimensionOrdinalMapping
+ .get(queryDimensions.get(i).getDimension().getOrdinal());
+ allProjectionListDimensionIndexes.add(dimensionOrdinal);
if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
addChildrenBlockIndex(allProjectionListDimensionIndexes,
queryDimensions.get(i).getDimension());
}
- if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
+ if (!filterDimensionOrdinal.contains(dimensionOrdinal)) {
blockIndex =
- dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
+ dimensionOrdinalToBlockMapping.get(dimensionOrdinal);
dimensionBlockIndex.add(blockIndex);
if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
@@ -249,6 +251,91 @@ public class QueryUtil {
}
/**
+ * This method will maintain a position mapping of query dimensions to current block dimensions
+ *
+ * @param queryDimensions
+ * @param currentBlockDimensions
+ * @return
+ */
+ public static Map<Integer, Integer> getQueryDimensionToCurrentBlockDimensionOrdinalMapping(
+ List<QueryDimension> queryDimensions, List<CarbonDimension> currentBlockDimensions) {
+ Map<Integer, Integer> queryToCurrentBlockDimensionOrdinals =
+ new HashMap<>(queryDimensions.size());
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
+ continue;
+ }
+ int ordinalOfDimensionFromCurrentBlock =
+ getOrdinalOfDimensionFromCurrentBlock(currentBlockDimensions,
+ queryDimension.getDimension());
+ queryToCurrentBlockDimensionOrdinals
+ .put(queryDimension.getDimension().getOrdinal(), ordinalOfDimensionFromCurrentBlock);
+ }
+ return queryToCurrentBlockDimensionOrdinals;
+ }
+
+ /**
+ * This method will return the key ordinal of the query dimension from the current block
+ *
+ * @param blockDimensions
+ * @param queryDimension
+ * @return
+ */
+ public static int getKeyOrdinalOfDimensionFromCurrentBlock(List<CarbonDimension> blockDimensions,
+ CarbonDimension queryDimension) {
+ int keyOrdinalInCurrentDimensionBlock = -1;
+ for (CarbonDimension blockDimension : blockDimensions) {
+ if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) {
+ keyOrdinalInCurrentDimensionBlock = blockDimension.getKeyOrdinal();
+ break;
+ }
+ }
+ return keyOrdinalInCurrentDimensionBlock;
+ }
+
+ /**
+ * This method will return the ordinal of the query dimension from the current block
+ *
+ * @param blockDimensions
+ * @param queryDimension
+ * @return
+ */
+ public static int getOrdinalOfDimensionFromCurrentBlock(List<CarbonDimension> blockDimensions,
+ CarbonDimension queryDimension) {
+ int keyOrdinalInCurrentDimensionBlock = -1;
+ for (CarbonDimension blockDimension : blockDimensions) {
+ if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) {
+ keyOrdinalInCurrentDimensionBlock = blockDimension.getOrdinal();
+ break;
+ }
+ }
+ return keyOrdinalInCurrentDimensionBlock;
+ }
+
+ /**
+ * This method will maintain a position mapping of query measures to current block measures
+ *
+ * @param queryMeasures
+ * @param currentBlockMeasures
+ * @return
+ */
+ public static Map<Integer, Integer> getQueryMeasuresToCurrentBlockMeasuresOrdinalMapping(
+ List<QueryMeasure> queryMeasures, List<CarbonMeasure> currentBlockMeasures) {
+ Map<Integer, Integer> queryToCurrentBlockDimensionOrdinals =
+ new HashMap<>(queryMeasures.size());
+ for (QueryMeasure queryMeasure : queryMeasures) {
+ for (CarbonMeasure currentBlockMeasure : currentBlockMeasures) {
+ if (queryMeasure.getMeasure().getColumnId().equals(currentBlockMeasure.getColumnId())) {
+ queryToCurrentBlockDimensionOrdinals
+ .put(queryMeasure.getMeasure().getOrdinal(), currentBlockMeasure.getOrdinal());
+ break;
+ }
+ }
+ }
+ return queryToCurrentBlockDimensionOrdinals;
+ }
+
+ /**
* Below method will be used to add the children block index
* this will be basically for complex dimension which will have children
*
@@ -400,18 +487,21 @@ public class QueryUtil {
*/
public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
+ Map<Integer, Integer> queryMeasuresToCurrentBlockMeasuresOrdinalMapping,
Set<CarbonMeasure> filterMeasures, List<Integer> allProjectionListMeasureIdexes) {
Set<Integer> measureBlockIndex = new HashSet<Integer>();
Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
for (int i = 0; i < queryMeasures.size(); i++) {
+ Integer measureOrdinal = queryMeasuresToCurrentBlockMeasuresOrdinalMapping
+ .get(queryMeasures.get(i).getMeasure().getOrdinal());
allProjectionListMeasureIdexes.add(queryMeasures.get(i).getMeasure().getOrdinal());
- if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) {
- measureBlockIndex
- .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
+ if (!filterMeasureOrdinal.contains(measureOrdinal)) {
+ measureBlockIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
}
}
for (int i = 0; i < expressionMeasure.size(); i++) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+ measureBlockIndex.add(queryMeasuresToCurrentBlockMeasuresOrdinalMapping
+ .get(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal())));
}
int[] measureIndexes =
ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
@@ -420,6 +510,54 @@ public class QueryUtil {
}
/**
+ * This method will create the updated list of filter measures present in the current block
+ *
+ * @param queryFilterMeasures
+ * @param currentBlockMeasures
+ * @return
+ */
+ public static Set<CarbonMeasure> getUpdatedFilterMeasures(Set<CarbonMeasure> queryFilterMeasures,
+ List<CarbonMeasure> currentBlockMeasures) {
+ if (!queryFilterMeasures.isEmpty()) {
+ Set<CarbonMeasure> updatedFilterMeasures = new HashSet<>(queryFilterMeasures.size());
+ for (CarbonMeasure queryMeasure : queryFilterMeasures) {
+ CarbonMeasure measureFromCurrentBlock =
+ CarbonUtil.getMeasureFromCurrentBlock(currentBlockMeasures, queryMeasure.getColumnId());
+ if (null != measureFromCurrentBlock) {
+ updatedFilterMeasures.add(measureFromCurrentBlock);
+ }
+ }
+ return updatedFilterMeasures;
+ } else {
+ return queryFilterMeasures;
+ }
+ }
+
+ /**
+ * This method will create the updated list of filter dimensions present in the current block
+ *
+ * @param queryFilterDimensions
+ * @param currentBlockDimensions
+ * @return
+ */
+ public static Set<CarbonDimension> getUpdatedFilterDimensions(
+ Set<CarbonDimension> queryFilterDimensions, List<CarbonDimension> currentBlockDimensions) {
+ if (!queryFilterDimensions.isEmpty()) {
+ Set<CarbonDimension> updatedFilterDimensions = new HashSet<>(queryFilterDimensions.size());
+ for (CarbonDimension queryDimension : queryFilterDimensions) {
+ CarbonDimension dimensionFromCurrentBlock =
+ CarbonUtil.getDimensionFromCurrentBlock(currentBlockDimensions, queryDimension);
+ if (null != dimensionFromCurrentBlock) {
+ updatedFilterDimensions.add(dimensionFromCurrentBlock);
+ }
+ }
+ return updatedFilterDimensions;
+ } else {
+ return queryFilterDimensions;
+ }
+ }
+
+ /**
* Below method will be used to get mapping whether dimension is present in
* order by or not
*
@@ -608,17 +746,20 @@ public class QueryUtil {
*/
public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
+ Map<Integer, Integer> queryDimensionToCurrentBlockDimensionOrdinalMapping,
Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
for (QueryDimension queryDimension : queryDimensions) {
if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
&& queryDimension.getDimension().numberOfChild() == 0) {
- dictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ dictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get(
+ queryDimensionToCurrentBlockDimensionOrdinalMapping
+ .get(queryDimension.getDimension().getOrdinal())));
} else if (
!CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
&& queryDimension.getDimension().numberOfChild() == 0) {
- noDictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ noDictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get(
+ queryDimensionToCurrentBlockDimensionOrdinalMapping
+ .get(queryDimension.getDimension().getOrdinal())));
}
}
}
@@ -759,8 +900,8 @@ public class QueryUtil {
if (filterDimension.hasEncoding(Encoding.IMPLICIT)) {
continue;
}
- fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
- eachComplexColumnValueSize, columnIdToDictionaryMap);
+ // fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
+ // eachComplexColumnValueSize, columnIdToDictionaryMap);
}
}
return complexTypeMap;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 3549932..847a795 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -16,16 +16,30 @@
*/
package org.apache.carbondata.core.scan.executor.util;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
+import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.spark.unsafe.types.UTF8String;
/**
* Utility class for restructuring
@@ -38,37 +52,160 @@ public class RestructureUtil {
* table blocks in that case we need to select only those dimension out of
* query dimension which is present in the current table block
*
+ * @param blockExecutionInfo
* @param queryDimensions
* @param tableBlockDimensions
+ * @param tableComplexDimension
* @return list of query dimension which is present in the table block
*/
- public static List<QueryDimension> getUpdatedQueryDimension(List<QueryDimension> queryDimensions,
+ public static List<QueryDimension> createDimensionInfoAndGetUpdatedQueryDimension(
+ BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions,
List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
List<QueryDimension> presentDimension =
new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ boolean[] isDimensionExists = new boolean[queryDimensions.size()];
+ Object[] defaultValues = new Object[queryDimensions.size()];
// selecting only those dimension which is present in the query
+ int dimIndex = 0;
for (QueryDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
presentDimension.add(queryDimension);
+ isDimensionExists[dimIndex] = true;
} else {
for (CarbonDimension tableDimension : tableBlockDimensions) {
- if (tableDimension.equals(queryDimension.getDimension())) {
+ if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
presentDimension.add(queryDimension);
+ isDimensionExists[dimIndex] = true;
+ break;
}
}
+ // add default value only in case query dimension is not found in the current block
+ if (!isDimensionExists[dimIndex]) {
+ defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
+ blockExecutionInfo.setRestructuredBlock(true);
+ }
}
+ dimIndex++;
}
- for (QueryDimension queryDimimension : queryDimensions) {
+ dimIndex = 0;
+ for (QueryDimension queryDimension : queryDimensions) {
for (CarbonDimension tableDimension : tableComplexDimension) {
- if (tableDimension.equals(queryDimimension.getDimension())) {
- presentDimension.add(queryDimimension);
+ if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
+ presentDimension.add(queryDimension);
+ isDimensionExists[dimIndex] = true;
+ break;
}
}
+ // add default value only in case query dimension is not found in the current block
+ if (!isDimensionExists[dimIndex]) {
+ defaultValues[dimIndex] =
+ validateAndGetDefaultValue(queryDimension.getDimension());
+ }
+ dimIndex++;
}
+ DimensionInfo dimensionInfo =
+ new DimensionInfo(isDimensionExists, defaultValues);
+ blockExecutionInfo.setDimensionInfo(dimensionInfo);
return presentDimension;
}
/**
+ * This method will validate and return the default value to be
+ * filled at the time of result preparation
+ *
+ * @param queryDimension
+ * @return
+ */
+ public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) {
+ byte[] defaultValue = queryDimension.getDefaultValue();
+ Object defaultValueToBeConsidered = null;
+ if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) {
+ // direct dictionary case
+ if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+ defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),
+ queryDimension.getDefaultValue());
+ } else {
+ // dictionary case
+ defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue);
+ }
+ } else {
+ // no dictionary
+ defaultValueToBeConsidered = getNoDictionaryDefaultValue(defaultValue);
+ }
+ return defaultValueToBeConsidered;
+ }
+
+ /**
+ * Method for computing default value for dictionary column
+ *
+ * @param defaultValue
+ * @return
+ */
+ private static Object getDictionaryDefaultValue(byte[] defaultValue) {
+ Object dictionaryDefaultValue = null;
+ // dictionary has 2 cases:
+ // 1. If default value is specified then its surrogate key will be 2
+ // 2. If default value is not specified then its surrogate key will be
+ // 1 which is for member default value null
+ if (isDefaultValueNull(defaultValue)) {
+ dictionaryDefaultValue = new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ } else {
+ dictionaryDefaultValue =
+ new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1);
+ }
+ return dictionaryDefaultValue;
+ }
+
+ /**
+ * Method for computing default value for direct dictionary
+ *
+ * @param dataType
+ * @param defaultValue
+ * @return
+ */
+ private static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
+ Object directDictionaryDefaultValue = null;
+ if (!isDefaultValueNull(defaultValue)) {
+ DirectDictionaryGenerator directDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);
+ if (directDictionaryGenerator != null) {
+ String value =
+ new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ directDictionaryDefaultValue =
+ directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value));
+ }
+ }
+ return directDictionaryDefaultValue;
+ }
+
+ /**
+ * Method for computing default value for no dictionary
+ *
+ * @param defaultValue
+ * @return
+ */
+ private static Object getNoDictionaryDefaultValue(byte[] defaultValue) {
+ Object noDictionaryDefaultValue = null;
+ if (!isDefaultValueNull(defaultValue)) {
+ noDictionaryDefaultValue = UTF8String.fromBytes(defaultValue);
+ }
+ return noDictionaryDefaultValue;
+ }
+
+ /**
+ * This method will validate whether a given value is empty or null
+ *
+ * @param defaultValue
+ * @return
+ */
+ private static boolean isDefaultValueNull(byte[] defaultValue) {
+ if (null == defaultValue) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
* Below method is to add dimension children for complex type dimension as
* internally we are creating dimension column for each each complex
* dimension so when complex query dimension request will come in the query,
@@ -102,37 +239,88 @@ public class RestructureUtil {
}
/**
- * Below method will be used to get the aggregator info object
+ * Method for computing measure default value based on the data type
+ *
+ * @param columnSchema
+ * @param defaultValue
+ * @return
+ */
+ public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) {
+ Object measureDefaultValue = null;
+ if (!isDefaultValueNull(defaultValue)) {
+ String value = null;
+ switch (columnSchema.getDataType()) {
+ case SHORT:
+ case INT:
+ case LONG:
+ value =
+ new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ measureDefaultValue = Long.parseLong(value);
+ break;
+ case DECIMAL:
+ BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);
+ if (columnSchema.getScale() > decimal.scale()) {
+ decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
+ }
+ measureDefaultValue = decimal;
+ break;
+ default:
+ value =
+ new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ Double parsedValue = Double.valueOf(value);
+ if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {
+ measureDefaultValue = value;
+ }
+ }
+ }
+ return measureDefaultValue;
+ }
+
+ /**
+ * Below method will be used to prepare the measure info object
* in this method some of the properties which will be extracted
* from query measure and current block measures will be set
*
+ * @param blockExecutionInfo
* @param queryMeasures measures present in query
* @param currentBlockMeasures current block measures
- * @return aggregator info
+ * @return measures present in the block
*/
- public static AggregatorInfo getAggregatorInfos(List<QueryMeasure> queryMeasures,
+ public static List<QueryMeasure> createMeasureInfoAndGetUpdatedQueryMeasures(
+ BlockExecutionInfo blockExecutionInfo, List<QueryMeasure> queryMeasures,
List<CarbonMeasure> currentBlockMeasures) {
- AggregatorInfo aggregatorInfos = new AggregatorInfo();
+ MeasureInfo measureInfo = new MeasureInfo();
+ List<QueryMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
int numberOfMeasureInQuery = queryMeasures.size();
- int[] measureOrdinals = new int[numberOfMeasureInQuery];
+ List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
Object[] defaultValues = new Object[numberOfMeasureInQuery];
boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
int index = 0;
for (QueryMeasure queryMeasure : queryMeasures) {
- measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal();
// if query measure exists in current dimension measures
// then setting measure exists is true
// otherwise adding a default value of a measure
- if (currentBlockMeasures.contains(queryMeasure.getMeasure())) {
- measureExistsInCurrentBlock[index] = true;
- } else {
- defaultValues[index] = queryMeasure.getMeasure().getDefaultValue();
+ for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
+ if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) {
+ presentMeasure.add(queryMeasure);
+ measureOrdinalList.add(carbonMeasure.getOrdinal());
+ measureExistsInCurrentBlock[index] = true;
+ break;
+ }
+ }
+ if (!measureExistsInCurrentBlock[index]) {
+ defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(),
+ queryMeasure.getMeasure().getDefaultValue());
+ blockExecutionInfo.setRestructuredBlock(true);
}
index++;
}
- aggregatorInfos.setDefaultValues(defaultValues);
- aggregatorInfos.setMeasureOrdinals(measureOrdinals);
- aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock);
- return aggregatorInfos;
+ int[] measureOrdinals =
+ ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()]));
+ measureInfo.setDefaultValues(defaultValues);
+ measureInfo.setMeasureOrdinals(measureOrdinals);
+ measureInfo.setMeasureExists(measureExistsInCurrentBlock);
+ blockExecutionInfo.setMeasureInfo(measureInfo);
+ return presentMeasure;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 520b460..deb07c7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -68,6 +69,8 @@ import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RestructureExcludeFilterExecutorImpl;
+import org.apache.carbondata.core.scan.filter.executer.RestructureIncludeFilterExecutorImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
@@ -164,7 +167,20 @@ public final class FilterUtil {
DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
- return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+ .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+ dimColResolvedFilterInfo.getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ // update dimension and column index according to the dimension position in current block
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+ dimColResolvedFilterInfo.getCopyObject();
+ dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+ dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties);
+ } else {
+ return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
+ segmentProperties);
+ }
} else {
return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
}
@@ -181,7 +197,20 @@ public final class FilterUtil {
DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
- return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+ .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+ dimColResolvedFilterInfo.getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ // update dimension and column index according to the dimension position in current block
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+ dimColResolvedFilterInfo.getCopyObject();
+ dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+ dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties);
+ } else {
+ return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
+ segmentProperties);
+ }
} else {
return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
}
@@ -571,6 +600,53 @@ public final class FilterUtil {
}
/**
+ * This method will check whether a default value for the non-existing column is present
+ * in the filter values list
+ *
+ * @param dimColumnEvaluatorInfo
+ * @return
+ */
+ public static boolean isDimensionDefaultValuePresentInFilterValues(
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
+ boolean isDefaultValuePresentInFilterValues = false;
+ DimColumnFilterInfo filterValues = dimColumnEvaluatorInfo.getFilterValues();
+ CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
+ byte[] defaultValue = dimension.getDefaultValue();
+ if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
+ // for no dictionary cases
+ // 3 cases: is NUll, is Not Null and filter on default value of newly added column
+ if (null == defaultValue) {
+ // default value for case where user gives is Null condition
+ defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
+ List<byte[]> noDictionaryFilterValuesList = filterValues.getNoDictionaryFilterValuesList();
+ for (byte[] filterValue : noDictionaryFilterValuesList) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue);
+ if (compare == 0) {
+ isDefaultValuePresentInFilterValues = true;
+ break;
+ }
+ }
+ } else {
+ // for dictionary and direct dictionary cases
+ // 3 cases: is NUll, is Not Null and filter on default value of newly added column
+ int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
+ if (null != defaultValue) {
+ defaultSurrogateValueToCompare++;
+ }
+ List<Integer> filterList = filterValues.getFilterList();
+ for (Integer filterValue : filterList) {
+ if (defaultSurrogateValueToCompare == filterValue) {
+ isDefaultValuePresentInFilterValues = true;
+ break;
+ }
+ }
+ }
+ return isDefaultValuePresentInFilterValues;
+ }
+
+ /**
* Below method will be used to covert the filter surrogate keys
* to mdkey
*
@@ -590,13 +666,14 @@ public final class FilterUtil {
int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
Arrays.fill(keys, 0);
- int[] rangesForMaskedByte =
- getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+ int keyOrdinalOfDimensionFromCurrentBlock = carbonDimension.getKeyOrdinal();
if (null != dimColumnFilterInfo) {
+ int[] rangesForMaskedByte =
+ getRangesForMaskedByte(keyOrdinalOfDimensionFromCurrentBlock, blockLevelKeyGenerator);
for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
try {
- if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
- keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ if (surrogate <= dimColumnsCardinality[keyOrdinalOfDimensionFromCurrentBlock]) {
+ keys[keyOrdinalOfDimensionFromCurrentBlock] = surrogate;
filterValuesList
.add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
} else {
@@ -647,12 +724,12 @@ public final class FilterUtil {
* @return long[] start key
*/
public static void getStartKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
- long[] startKey, List<long[]> startKeyList) {
+ SegmentProperties segmentProperties, long[] startKey, List<long[]> startKeyList) {
for (int i = 0; i < startKey.length; i++) {
// The min surrogate key is 1, set it as the init value for starkey of each column level
startKey[i] = 1;
}
- getStartKeyWithFilter(dimensionFilter, startKey, startKeyList);
+ getStartKeyWithFilter(dimensionFilter, segmentProperties, startKey, startKeyList);
}
/**
@@ -673,6 +750,7 @@ public final class FilterUtil {
*/
public static void getStartKeyForNoDictionaryDimension(
DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties,
SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
@@ -692,18 +770,25 @@ public final class FilterUtil {
if (isExcludePresent) {
continue;
}
+ // in case of restructure scenarios it can happen that the filter dimension is not
+ // present in the current block. In those cases no need to determine the key
+ CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+ .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey());
+ if (null == dimensionFromCurrentBlock) {
+ continue;
+ }
// step 2
byte[] noDictionaryStartKey =
listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0);
if (setOfStartKeyByteArray.isEmpty()) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
- } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
+ } else if (null == setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) {
+ setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
} else if (ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()),
+ .compareTo(setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()),
noDictionaryStartKey) > 0) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
}
}
}
@@ -727,6 +812,7 @@ public final class FilterUtil {
*/
public static void getEndKeyForNoDictionaryDimension(
DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties,
SortedMap<Integer, byte[]> setOfEndKeyByteArray) {
Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
@@ -747,18 +833,25 @@ public final class FilterUtil {
if (isExcludePresent) {
continue;
}
+ // in case of restructure scenarios it can happen that the filter dimension is not
+ // present in the current block. In those cases no need to determine the key
+ CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+ .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey());
+ if (null == dimensionFromCurrentBlock) {
+ continue;
+ }
// step 2
byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList()
.get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1);
if (setOfEndKeyByteArray.isEmpty()) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
- } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
+ } else if (null == setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) {
+ setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
} else if (ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey)
- < 0) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ .compareTo(setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()),
+ noDictionaryEndKey) < 0) {
+ setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
}
}
@@ -794,8 +887,8 @@ public final class FilterUtil {
* @param startKey
*/
private static void getStartKeyWithFilter(
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] startKey,
- List<long[]> startKeyList) {
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+ SegmentProperties segmentProperties, long[] startKey, List<long[]> startKeyList) {
for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
List<DimColumnFilterInfo> values = entry.getValue();
if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
@@ -810,9 +903,17 @@ public final class FilterUtil {
if (isExcludePresent) {
continue;
}
+ int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil
+ .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+ entry.getKey());
+ // if key ordinal is -1 that means this dimension does not exist in the current block.
+ // Applicable for restructure scenarios
+ if (keyOrdinalOfDimensionFromCurrentBlock == -1) {
+ continue;
+ }
for (DimColumnFilterInfo info : values) {
- if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) {
- startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0);
+ if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+ startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
}
}
long[] newStartKey = new long[startKey.length];
@@ -831,7 +932,7 @@ public final class FilterUtil {
endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i),
segmentProperties.getDimColumnsCardinality());
}
- getEndKeyWithFilter(dimensionFilter, endKey, endKeyList);
+ getEndKeyWithFilter(dimensionFilter, segmentProperties, endKey, endKeyList);
}
@@ -850,8 +951,8 @@ public final class FilterUtil {
}
private static void getEndKeyWithFilter(
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] endKey,
- List<long[]> endKeyList) {
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+ SegmentProperties segmentProperties, long[] endKey, List<long[]> endKeyList) {
for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
List<DimColumnFilterInfo> values = entry.getValue();
if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
@@ -866,11 +967,18 @@ public final class FilterUtil {
if (isExcludeFilterPresent) {
continue;
}
-
+ int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil
+ .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+ entry.getKey());
+ // if key ordinal is -1 that means this dimension does not exist in the current block.
+ // Applicable for restructure scenarios
+ if (keyOrdinalOfDimensionFromCurrentBlock == -1) {
+ continue;
+ }
for (DimColumnFilterInfo info : values) {
- if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList()
+ if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
.get(info.getFilterList().size() - 1)) {
- endKey[entry.getKey().getKeyOrdinal()] =
+ endKey[keyOrdinalOfDimensionFromCurrentBlock] =
info.getFilterList().get(info.getFilterList().size() - 1);
}
}
@@ -1291,7 +1399,8 @@ public final class FilterUtil {
traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(),
segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
startKeyList, endKeyList);
- filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList);
+ filterResolverTree
+ .getStartKey(segmentProperties, startKeys, setOfStartKeyByteArray, startKeyList);
filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray,
endKeyList);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
new file mode 100644
index 0000000..68f0dfe
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class RestructureExcludeFilterExecutorImpl implements FilterExecuter {
+
+ protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ protected SegmentProperties segmentProperties;
+
+ /**
+ * flag to check whether filter values contain the default value applied on the dimension column
+ * which does not exist in the current block
+ */
+ protected boolean isDefaultValuePresentInFilterValues;
+
+ public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
+ SegmentProperties segmentProperties) {
+ this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+ this.segmentProperties = segmentProperties;
+ isDefaultValuePresentInFilterValues =
+ FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColEvaluatorInfo);
+ }
+
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ BitSetGroup bitSetGroup = new BitSetGroup(1);
+ BitSet bitSet = new BitSet(numberOfRows);
+ bitSet.set(0, numberOfRows, !isDefaultValuePresentInFilterValues);
+ bitSetGroup.setBitSet(bitSet, 0);
+ return bitSetGroup;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.flip(0, 1);
+ return bitSet;
+ }
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
new file mode 100644
index 0000000..c1ba2af
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class RestructureIncludeFilterExecutorImpl implements FilterExecuter {
+
+ protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+ protected SegmentProperties segmentProperties;
+
+ /**
+ * flag to check whether filter values contain the default value applied on the dimension column
+ * which does not exist in the current block
+ */
+ protected boolean isDefaultValuePresentInFilterValues;
+
+ public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
+ SegmentProperties segmentProperties) {
+ this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+ this.segmentProperties = segmentProperties;
+ isDefaultValuePresentInFilterValues =
+ FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColumnEvaluatorInfo);
+ }
+
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ BitSetGroup bitSetGroup = new BitSetGroup(1);
+ BitSet bitSet = new BitSet(numberOfRows);
+ bitSet.set(0, numberOfRows, isDefaultValuePresentInFilterValues);
+ bitSetGroup.setBitSet(bitSet, 0);
+ return bitSetGroup;
+ }
+
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.set(0, isDefaultValuePresentInFilterValues);
+ return bitSet;
+ }
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 7595ab6..b48382e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -19,7 +19,10 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -40,8 +43,11 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
@@ -68,29 +74,85 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
/**
* it has index at which given dimension is stored in file
*/
- private int[] blocksIndex;
+ protected int[] dimensionBlocksIndex;
+
+ /**
+ * it has index at which given measure is stored in file
+ */
+ protected int[] measureBlocksIndex;
private Map<Integer, GenericQueryType> complexDimensionInfoMap;
+ /**
+ * flag to check whether the filter dimension is present in current block list of dimensions.
+ * Applicable for restructure scenarios
+ */
+ protected boolean[] isDimensionPresentInCurrentBlock;
+
+ /**
+ * flag to check whether the filter measure is present in current block list of measures.
+ * Applicable for restructure scenarios
+ */
+ protected boolean[] isMeasurePresentInCurrentBlock;
+
public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
Map<Integer, GenericQueryType> complexDimensionInfoMap) {
this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
this.segmentProperties = segmentProperties;
- this.blocksIndex = new int[dimColEvaluatorInfoList.size()];
- for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
- this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(i).getColumnIndex());
- }
+ this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
} else {
this.msrColEvalutorInfoList = msrColEvalutorInfoList;
}
+ this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
this.exp = exp;
this.tableIdentifier = tableIdentifier;
this.complexDimensionInfoMap = complexDimensionInfoMap;
+ initDimensionBlockIndexes();
+ initMeasureBlockIndexes();
+ }
+
+ /**
+ * This method will initialize the dimension info for the current block to be
+ * used for filtering the data
+ */
+ private void initDimensionBlockIndexes() {
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ // find the dimension in the current block dimensions list
+ CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+ .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+ dimColEvaluatorInfoList.get(i).getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ this.dimensionBlocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimensionFromCurrentBlock.getOrdinal());
+ isDimensionPresentInCurrentBlock[i] = true;
+ }
+ }
+ }
+
+ /**
+ * This method will initialize the measure info for the current block to be
+ * used for filtering the data
+ */
+ private void initMeasureBlockIndexes() {
+ for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
+ // find the measure in the current block measures list
+ CarbonMeasure measureFromCurrentBlock = CarbonUtil
+ .getMeasureFromCurrentBlock(segmentProperties.getMeasures(),
+ msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId());
+ if (null != measureFromCurrentBlock) {
+ msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal());
+ this.measureBlocksIndex[i] = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ .get(measureFromCurrentBlock.getOrdinal());
+ isMeasurePresentInCurrentBlock[i] = true;
+ }
+ }
}
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
@@ -102,13 +164,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
int pageNumbers = 0;
if (dimColEvaluatorInfoList.size() > 0) {
- pageNumbers = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getPagesCount();
- numberOfRows = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getRowCount();
+ if (isDimensionPresentInCurrentBlock[0]) {
+ pageNumbers =
+ blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount();
+ numberOfRows =
+ blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount();
+ }
}
if (msrColEvalutorInfoList.size() > 0) {
- int columnIndex = msrColEvalutorInfoList.get(0).getColumnIndex();
- pageNumbers = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getPagesCount();
- numberOfRows = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getRowCount();
+ if (isMeasurePresentInCurrentBlock[0]) {
+ pageNumbers =
+ blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount();
+ numberOfRows =
+ blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount();
+ }
}
BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
for (int i = 0; i < pageNumbers; i++) {
@@ -152,13 +221,21 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
String memberString;
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+ // if filter dimension is not present in the current add its default value
+ if (!isDimensionPresentInCurrentBlock[i]) {
+ // fill default value here
+ record[dimColumnEvaluatorInfo.getRowIndex()] =
+ getDimensionDefaultValue(dimColumnEvaluatorInfo);
+ continue;
+ }
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
&& dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
- record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
+ record[dimColumnEvaluatorInfo.getRowIndex()] =
+ dimColumnEvaluatorInfo.getDimension().getDefaultValue();
}
DimensionColumnDataChunk columnDataChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]
+ blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]
.convertToDimColDataChunk(pageIndex);
if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& columnDataChunk instanceof VariableLengthDimensionDataChunk) {
@@ -178,7 +255,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex,
- dimColumnEvaluatorInfo, blocksIndex[i]);
+ dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
@@ -196,7 +273,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
try {
- GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
complexType.parseBlocksAndReturnComplexColumnByteArray(
@@ -211,8 +288,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
DataType msrType;
-
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+ for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
+ MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
switch (msrColumnEvalutorInfo.getType()) {
case INT:
case LONG:
@@ -224,10 +301,19 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
default:
msrType = DataType.DOUBLE;
}
- // if measure doesnt exist then set the default value.
+ // add default value for the measure in case filter measure is not present
+ // in the current block measure list
+ if (!isMeasurePresentInCurrentBlock[i]) {
+ byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
+ record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil
+ .getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(),
+ defaultValue);
+ continue;
+ }
+
Object msrValue;
MeasureColumnDataChunk measureColumnDataChunk =
- blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
.convertToMeasureColDataChunk(pageIndex);
switch (msrType) {
case INT:
@@ -236,8 +322,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
break;
case DECIMAL:
- msrValue = measureColumnDataChunk.getMeasureDataHolder()
- .getReadableBigDecimalValueByIndex(index);
+ BigDecimal bigDecimalValue =
+ measureColumnDataChunk.getMeasureDataHolder()
+ .getReadableBigDecimalValueByIndex(index);
+ if (null != bigDecimalValue
+ && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale()
+ > bigDecimalValue.scale()) {
+ bigDecimalValue = bigDecimalValue
+ .setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(),
+ RoundingMode.HALF_UP);
+ }
+ msrValue = bigDecimalValue;
break;
default:
msrValue =
@@ -250,6 +345,28 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
/**
+ * This method will compute the default value for a dimension
+ *
+ * @param dimColumnEvaluatorInfo
+ * @return
+ */
+ private Object getDimensionDefaultValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
+ Object dimensionDefaultValue = null;
+ CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
+ if (dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
+ .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ byte[] defaultValue = dimension.getDefaultValue();
+ if (null != defaultValue) {
+ dimensionDefaultValue =
+ new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
+ } else {
+ dimensionDefaultValue = RestructureUtil.validateAndGetDefaultValue(dimension);
+ }
+ return dimensionDefaultValue;
+ }
+
+ /**
* method will read the actual data from the direct dictionary generator
* by passing direct dictionary value.
*
@@ -329,9 +446,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
byte[] colData = chunk.getChunkData(index);
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
- QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
int dictionaryValue = (int) result[segmentProperties
- .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())];
+ .getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
return dictionaryValue;
} catch (KeyGenException e) {
LOGGER.error(e);
@@ -346,30 +463,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
+ /**
+ * This method will set the bitset to true by default for a given number of rows
+ *
+ * @param numberOfRows
+ * @return
+ */
+ protected BitSetGroup getDefaultBitSetGroup(int numberOfRows) {
+ BitSetGroup bitSetGroup = new BitSetGroup(1);
+ BitSet bitSet = new BitSet(numberOfRows);
+ bitSet.set(0, numberOfRows, true);
+ bitSetGroup.setBitSet(bitSet, 0);
+ return bitSetGroup;
+ }
+
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
&& dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]) {
- blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]] =
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
+ blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
+ .getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]);
}
} else {
- GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+ GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
complexType.fillRequiredBlockData(blockChunkHolder);
}
}
if (null != msrColEvalutorInfoList) {
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo
- .getColumnIndex()]) {
- blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
- blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
- msrColumnEvalutorInfo.getColumnIndex());
- }
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) {
+ blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] =
+ blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 9f28d7c..e748141 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -38,6 +39,11 @@ import org.apache.carbondata.core.util.CarbonUtil;
public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
+ /**
+ * flag to check whether default values is present in the filter value list
+ */
+ private boolean isDefaultValuePresentInFilter;
+
public RowLevelRangeGrtThanFiterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -46,44 +52,71 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
+ checkIfDefaultValueIsPresentInFilterList();
+ }
+
+ /**
+ * This method will check whether default value is present in the given filter values
+ */
+ private void checkIfDefaultValueIsPresentInFilterList() {
+ if (!this.isDimensionPresentInCurrentBlock[0]) {
+ CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
+ byte[] defaultValue = dimension.getDefaultValue();
+ if (null != defaultValue) {
+ for (int k = 0; k < filterRangeValues.length; k++) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ if (maxCompare < 0) {
+ isDefaultValuePresentInFilter = true;
+ break;
+ }
+ }
+ }
+ }
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- byte[][] filterValues = this.filterRangeValues;
- int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
- boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues);
+ boolean isScanRequired =
+ isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
if (isScanRequired) {
bitSet.set(0);
}
return bitSet;
-
}
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
- // if any filter value is in range than this block needs to be
- // scanned means always less than block max range.
- if (maxCompare < 0) {
- isScanRequired = true;
- break;
+ if (isDimensionPresentInCurrentBlock[0]) {
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned means always less than block max range.
+ if (maxCompare < 0) {
+ isScanRequired = true;
+ break;
+ }
}
+ } else {
+ isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock[0]) {
+ return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize());
+ }
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ .get(dimensionBlocksIndex[0]);
if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
@@ -238,14 +271,15 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- super.readBlocks(blockChunkHolder);
- }
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (isDimensionPresentInCurrentBlock[0]) {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ super.readBlocks(blockChunkHolder);
+ }
+ int blockIndex = dimensionBlocksIndex[0];
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
}
}
}