You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:09:15 UTC
[30/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
new file mode 100644
index 0000000..fd66555
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.expression.exception;
+
+import java.util.Locale;
+
+public class FilterUnsupportedException extends Exception {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param errorCode The error code for this exception.
+ * @param msg The error message for this exception.
+ */
+ public FilterUnsupportedException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param errorCode The error code for this exception.
+ * @param msg The error message for this exception.
+ */
+ public FilterUnsupportedException(String msg, Throwable t) {
+ super(msg, t);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param errorCode The error code for this exception.
+ * @param msg The error message for this exception.
+ */
+ public FilterUnsupportedException(Throwable t) {
+ super(t);
+ }
+
+ /**
+ * This method is used to get the localized message.
+ *
+ * @param locale - A Locale object represents a specific geographical,
+ * political, or cultural region.
+ * @return - Localized error message.
+ */
+ public String getLocalizedMessage(Locale locale) {
+ return "";
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
new file mode 100644
index 0000000..79120e5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.expression.logical;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.ExpressionResult;
+import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.scan.filter.intf.RowIntf;
+
+public class AndExpression extends BinaryLogicalExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ public AndExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult resultLeft = left.evaluate(value);
+ ExpressionResult resultRight = right.evaluate(value);
+ switch (resultLeft.getDataType()) {
+ case BOOLEAN:
+ resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() && resultRight.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying AND Expression Filter");
+ }
+ return resultLeft;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return ExpressionType.AND;
+ }
+
+ @Override public String getString() {
+ // TODO Auto-generated method stub
+ return "And(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
new file mode 100644
index 0000000..1b4a0fc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.expression.logical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.scan.expression.BinaryExpression;
+import org.apache.carbondata.scan.expression.ColumnExpression;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.ExpressionResult;
+import org.apache.carbondata.scan.expression.LiteralExpression;
+
+public abstract class BinaryLogicalExpression extends BinaryExpression {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public BinaryLogicalExpression(Expression left, Expression right) {
+ super(left, right);
+ // TODO Auto-generated constructor stub
+ }
+
+ public List<ExpressionResult> getLiterals() {
+ List<ExpressionResult> listOfExp =
+ new ArrayList<ExpressionResult>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getExpressionResultList(this, listOfExp);
+ Collections.sort(listOfExp);
+ return listOfExp;
+ }
+
+ // Will get the column informations involved in the expressions by
+ // traversing the tree
+ public List<ColumnExpression> getColumnList() {
+ // TODO
+ List<ColumnExpression> listOfExp =
+ new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getColumnList(this, listOfExp);
+ return listOfExp;
+ }
+
+ private void getColumnList(Expression expression, List<ColumnExpression> lst) {
+ if (expression instanceof ColumnExpression) {
+ ColumnExpression colExp = (ColumnExpression) expression;
+ boolean found = false;
+
+ for (ColumnExpression currentColExp : lst) {
+ if (currentColExp.getColumnName().equals(colExp.getColumnName())) {
+ found = true;
+ colExp.setColIndex(currentColExp.getColIndex());
+ break;
+ }
+ }
+ if (!found) {
+ colExp.setColIndex(lst.size());
+ lst.add(colExp);
+ }
+ }
+ for (Expression child : expression.getChildren()) {
+ getColumnList(child, lst);
+ }
+ }
+
+ public boolean isSingleDimension() {
+ List<ColumnExpression> listOfExp =
+ new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getColumnList(this, listOfExp);
+ if (listOfExp.size() == 1 && listOfExp.get(0).isDimension()) {
+ return true;
+ }
+ return false;
+
+ }
+
+ private void getExpressionResultList(Expression binaryConditionalExpression,
+ List<ExpressionResult> listOfExp) {
+ if (binaryConditionalExpression instanceof LiteralExpression) {
+ ExpressionResult colExp =
+ ((LiteralExpression) binaryConditionalExpression).getExpressionResult();
+ listOfExp.add(colExp);
+ }
+ for (Expression child : binaryConditionalExpression.getChildren()) {
+ getExpressionResultList(child, listOfExp);
+ }
+
+ }
+
+ /**
+ * the method will return flag (true or false) depending on the existence of the
+ * direct dictionary columns in conditional expression
+ *
+ * @return the method will return flag (true or false)
+ */
+ public boolean isDirectDictionaryColumns() {
+ List<ColumnExpression> listOfExp =
+ new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getColumnList(this, listOfExp);
+ for (ColumnExpression ce : listOfExp) {
+ if (!ce.getCarbonColumn().hasEncoding(Encoding.DICTIONARY)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/expression/logical/NotExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/NotExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/NotExpression.java
new file mode 100644
index 0000000..2be732b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/logical/NotExpression.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.expression.logical;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.ExpressionResult;
+import org.apache.carbondata.scan.expression.UnaryExpression;
+import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.scan.filter.intf.RowIntf;
+
+public class NotExpression extends UnaryExpression {
+ private static final long serialVersionUID = 1L;
+
+ public NotExpression(Expression child) {
+ super(child);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ ExpressionResult expResult = child.evaluate(value);
+ expResult.set(DataType.BOOLEAN, !(expResult.getBoolean()));
+ switch (expResult.getDataType()) {
+ case BOOLEAN:
+ expResult.set(DataType.BOOLEAN, !(expResult.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying NOT Expression Filter");
+ }
+ return expResult;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.NOT;
+ }
+
+ @Override public String getString() {
+ return "Not(" + child.getString() + ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
new file mode 100644
index 0000000..4723fb8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.expression.logical;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.ExpressionResult;
+import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.scan.filter.intf.RowIntf;
+
+public class OrExpression extends BinaryLogicalExpression {
+
+ private static final long serialVersionUID = 4220598043176438380L;
+
+ public OrExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ ExpressionResult resultLeft = left.evaluate(value);
+ ExpressionResult resultRight = right.evaluate(value);
+ switch (resultLeft.getDataType()) {
+ case BOOLEAN:
+ resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() || resultRight.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying OR Expression Filter");
+ }
+
+ return resultLeft;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.OR;
+ }
+
+ @Override public String getString() {
+ return "Or(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
new file mode 100644
index 0000000..0d51286
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.filter;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class DimColumnFilterInfo implements Serializable {
+
+ private static final long serialVersionUID = 8181578747306832771L;
+
+ private boolean isIncludeFilter;
+
+ private List<Integer> filterList;
+
+ /**
+ * maintain the no dictionary filter values list.
+ */
+ private List<byte[]> noDictionaryFilterValuesList;
+
+ public List<byte[]> getNoDictionaryFilterValuesList() {
+ return noDictionaryFilterValuesList;
+ }
+
+ public boolean isIncludeFilter() {
+ return isIncludeFilter;
+ }
+
+ public void setIncludeFilter(boolean isIncludeFilter) {
+ this.isIncludeFilter = isIncludeFilter;
+ }
+
+ public List<Integer> getFilterList() {
+ return filterList;
+ }
+
+ public void setFilterList(List<Integer> filterList) {
+ this.filterList = filterList;
+ }
+
+ public void setFilterListForNoDictionaryCols(List<byte[]> noDictionaryFilterValuesList) {
+ this.noDictionaryFilterValuesList = noDictionaryFilterValuesList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
new file mode 100644
index 0000000..b17dd3e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.scan.filter;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.scan.expression.BinaryExpression;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.scan.filter.resolver.ConditionalFilterResolverImpl;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.scan.filter.resolver.LogicalFilterResolverImpl;
+import org.apache.carbondata.scan.filter.resolver.RowLevelFilterResolverImpl;
+import org.apache.carbondata.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
+
+public class FilterExpressionProcessor implements FilterProcessor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(FilterExpressionProcessor.class.getName());
+
+ /**
+ * Implementation will provide the resolved form of filters based on the
+ * filter expression tree which is been passed in Expression instance.
+ *
+ * @param expressionTree , filter expression tree
+ * @param tableIdentifier ,contains carbon store informations
+ * @return a filter resolver tree
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ public FilterResolverIntf getFilterResolver(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ if (null != expressionTree && null != tableIdentifier) {
+ return getFilterResolvertree(expressionTree, tableIdentifier);
+ }
+ return null;
+ }
+
+ /**
+ * This API will scan the Segment level all btrees and selects the required
+ * block reference nodes inorder to push the same to executer for applying filters
+ * on the respective data reference node.
+ * Following Algorithm is followed in below API
+ * Step:1 Get the start end key based on the filter tree resolver information
+ * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
+ * node(block)
+ * Step:3 Once data reference node ranges retrieved traverse the node within this range
+ * and select the node based on the block min and max value and the filter value.
+ * Step:4 The selected blocks will be send to executers for applying the filters with the help
+ * of Filter executers.
+ *
+ * @throws QueryExecutionException
+ */
+ public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
+ FilterResolverIntf filterResolver, AbstractIndex tableSegment,
+ AbsoluteTableIdentifier tableIdentifier) throws QueryExecutionException {
+ // Need to get the current dimension tables
+ List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
+ // getting the start and end index key based on filter for hitting the
+ // selected block reference nodes based on filter resolver tree.
+ LOGGER.debug("preparing the start and end key for finding"
+ + "start and end block as per filter resolver");
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
+ tableIdentifier, filterResolver, listOfStartEndKeys);
+ // reading the first value from list which has start key
+ IndexKey searchStartKey = listOfStartEndKeys.get(0);
+ // reading the last value from list which has end key
+ IndexKey searchEndKey = listOfStartEndKeys.get(1);
+ if (null == searchStartKey && null == searchEndKey) {
+ try {
+ // TODO need to handle for no dictionary dimensions
+ searchStartKey =
+ FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
+ // TODO need to handle for no dictionary dimensions
+ searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
+ } catch (KeyGenException e) {
+ return listOfDataBlocksToScan;
+ }
+ }
+
+ LOGGER.debug(
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
+ + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
+ + searchEndKey.getNoDictionaryKeys());
+ long startTimeInMillis = System.currentTimeMillis();
+ DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
+ DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(),null);
+ while (startBlock != endBlock) {
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock,
+ tableSegment.getSegmentProperties());
+ startBlock = startBlock.getNextDataRefNode();
+ }
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock,
+ tableSegment.getSegmentProperties());
+ LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (
+ System.currentTimeMillis() - startTimeInMillis)
+ + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan
+ .size());
+
+ return listOfDataBlocksToScan;
+ }
+
+ /**
+ * Selects the blocks based on col max and min value.
+ *
+ * @param filterResolver
+ * @param listOfDataBlocksToScan
+ * @param dataRefNode
+ * @param segmentProperties
+ */
+ private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter,
+ List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode,
+ SegmentProperties segmentProperties) {
+
+ BitSet bitSet = filterExecuter
+ .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue());
+ if (!bitSet.isEmpty()) {
+ listOfDataBlocksToScan.add(dataRefNode);
+
+ }
+ }
+
+ /**
+ * API will return a filter resolver instance which will be used by
+ * executers to evaluate or execute the filters.
+ *
+ * @param expressionTree , resolver tree which will hold the resolver tree based on
+ * filter expression.
+ * @return FilterResolverIntf type.
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ private FilterResolverIntf getFilterResolvertree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ FilterResolverIntf filterEvaluatorTree =
+ createFilterResolverTree(expressionTree, tableIdentifier, null);
+ traverseAndResolveTree(filterEvaluatorTree, tableIdentifier);
+ return filterEvaluatorTree;
+ }
+
+ /**
+ * constructing the filter resolver tree based on filter expression.
+ * this method will visit each node of the filter resolver and prepares
+ * the surrogates of the filter members which are involved filter
+ * expression.
+ *
+ * @param filterResolverTree
+ * @param tableIdentifier
+ * @throws FilterUnsupportedException
+ * @throws QueryExecutionException
+ */
+ private void traverseAndResolveTree(FilterResolverIntf filterResolverTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException {
+ if (null == filterResolverTree) {
+ return;
+ }
+ traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier);
+
+ filterResolverTree.resolve(tableIdentifier);
+
+ traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier);
+ }
+
+ /**
+ * Pattern used : Visitor Pattern
+ * Method will create filter resolver tree based on the filter expression tree,
+ * in this algorithm based on the expression instance the resolvers will created
+ *
+ * @param expressionTree
+ * @param tableIdentifier
+ * @return
+ */
+ private FilterResolverIntf createFilterResolverTree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier, Expression intermediateExpression) {
+ ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
+ BinaryExpression currentExpression = null;
+ switch (filterExpressionType) {
+ case OR:
+ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
+ currentExpression),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
+ currentExpression),currentExpression);
+ case AND:
+ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
+ currentExpression),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
+ currentExpression), currentExpression);
+ case EQUALS:
+ case IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, false, expressionTree,
+ tableIdentifier, expressionTree);
+ case GREATERTHAN:
+ case GREATERTHAN_EQUALTO:
+ case LESSTHAN:
+ case LESSTHAN_EQUALTO:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, true, expressionTree,
+ tableIdentifier, expressionTree);
+
+ case NOT_EQUALS:
+ case NOT_IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.NOT_EQUALS, false,
+ expressionTree, tableIdentifier, expressionTree);
+
+ default:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.UNKNOWN, false, expressionTree,
+ tableIdentifier, expressionTree);
+ }
+ }
+
+ /**
+ * Factory method which will return the resolver instance based on filter expression
+ * expressions.
+ */
+ private FilterResolverIntf getFilterResolverBasedOnExpressionType(
+ ExpressionType filterExpressionType, boolean isExpressionResolve, Expression expression,
+ AbsoluteTableIdentifier tableIdentifier, Expression expressionTree) {
+ BinaryConditionalExpression currentCondExpression = null;
+ ConditionalExpression condExpression = null;
+ switch (filterExpressionType) {
+ case EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ // getting new dim index.
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.GREATERTHAN_EQUALTO
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.LESSTHAN_EQUALTO) {
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true);
+
+ }
+ break;
+ case NOT_EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+ if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
+
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ break;
+ default:
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.isSingleDimension()
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new ConditionalFilterResolverImpl(expression, true, true);
+ } else {
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+ } else {
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+ }
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
new file mode 100644
index 0000000..35948e3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
@@ -0,0 +1,60 @@
+package org.apache.carbondata.scan.filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
+import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+
+public interface FilterProcessor {
+
+ /**
+ * API will provide the resolved form of filters based on the filter
+ * expression tree which is been passed.
+ *
+ * @param expressionTree , filter expression tree
+ * @param tableIdentifier ,contains carbon store informations.
+ * @return
+ * @throws QueryExecutionException
+ * @throws FilterUnsupportedException
+ */
+ FilterResolverIntf getFilterResolver(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException;
+
+ /**
+ * This API is exposed inorder to get the required block reference node
+ * based on the filter.The block list will be send to the executer tasks inorder
+ * to apply filters.
+ *
+ * @param filterResolver DataBlock list with resolved filters
+ * @return list of DataRefNode.
+ * @throws QueryExecutionException
+ */
+ List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
+ AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier)
+ throws QueryExecutionException;
+
+}