You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2012/10/18 02:28:21 UTC
svn commit: r1399491 - in
/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery:
compiler/rewriter/ compiler/rewriter/rules/ metadata/ xmlquery/query/
Author: prestonc
Date: Thu Oct 18 00:28:21 2012
New Revision: 1399491
URL: http://svn.apache.org/viewvc?rev=1399491&view=rev
Log:
Added a query plan rewrite rule for fn:collection. The new plan alerts algebricks of partitioned data.
Added:
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java (with props)
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java (with props)
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java (with props)
Modified:
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java?rev=1399491&r1=1399490&r2=1399491&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java Thu Oct 18 00:28:21 2012
@@ -19,6 +19,8 @@ package org.apache.vxquery.compiler.rewr
import java.util.LinkedList;
import java.util.List;
+import org.apache.vxquery.compiler.rewriter.rules.CollectionRewriteRule;
+
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.BreakSelectIntoConjunctsRule;
@@ -48,6 +50,12 @@ import edu.uci.ics.hyracks.algebricks.re
import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetExecutionModeRule;
public class RewriteRuleset {
+ public final static List<IAlgebraicRewriteRule> buildXQueryNormalizationRuleCollection() {
+ List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
+ normalization.add(new CollectionRewriteRule());
+ return normalization;
+ }
+
public final static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
List<IAlgebraicRewriteRule> typeInfer = new LinkedList<IAlgebraicRewriteRule>();
typeInfer.add(new InferTypesRule());
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java?rev=1399491&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java Thu Oct 18 00:28:21 2012
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.compiler.rewriter.rules;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+
+public class CollectionFileDomain implements INodeDomain {
+
+ private String collectionName;
+
+ public CollectionFileDomain(String collectionName) {
+ this.collectionName = collectionName;
+ }
+
+ @Override
+ public Integer cardinality() {
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return "CollectionFileDomain: " + collectionName;
+ }
+
+ @Override
+ public boolean sameAs(INodeDomain domain) {
+ if (!(domain instanceof CollectionFileDomain)) {
+ return false;
+ }
+ CollectionFileDomain cfd = (CollectionFileDomain) domain;
+ if (collectionName.compareTo(cfd.collectionName) != 0) {
+ return false;
+ }
+ return true;
+ }
+
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionFileDomain.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java?rev=1399491&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java Thu Oct 18 00:28:21 2012
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.compiler.rewriter.rules;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.vxquery.compiler.algebricks.VXQueryConstantValue;
+import org.apache.vxquery.datamodel.accessors.SequencePointable;
+import org.apache.vxquery.datamodel.accessors.TaggedValuePointable;
+import org.apache.vxquery.datamodel.values.ValueTag;
+import org.apache.vxquery.functions.BuiltinFunctions;
+import org.apache.vxquery.metadata.VXQueryCollectionDataSource;
+import org.apache.vxquery.types.AnyItemType;
+import org.apache.vxquery.types.BuiltinTypeRegistry;
+import org.apache.vxquery.types.Quantifier;
+import org.apache.vxquery.types.SequenceType;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+
+public class CollectionRewriteRule implements IAlgebraicRewriteRule {
+ final ByteBufferInputStream bbis = new ByteBufferInputStream();
+ final DataInputStream di = new DataInputStream(bbis);
+ final UTF8StringPointable stringp = (UTF8StringPointable) UTF8StringPointable.FACTORY.createPointable();
+ final TaggedValuePointable tvp = (TaggedValuePointable) TaggedValuePointable.FACTORY.createPointable();
+ final SequencePointable seqp = (SequencePointable) SequencePointable.FACTORY.createPointable();
+
+ /**
+ * Find the default query plan created for collection and updated it to use parallelization.
+ * The following is an example of of the operators we are looking with a constant for the collection name.
+ * Search pattern: unnest <- assign [function-call: collection] <- assign [constant: string]
+ */
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+ return false;
+ }
+ UnnestOperator unnest = (UnnestOperator) op;
+
+ // Check if assign is for fn:Collection.
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ AssignOperator assign = (AssignOperator) op2;
+
+ // Check to see if the expression is a function and fn:Collection.
+ ILogicalExpression logicalExpression = (ILogicalExpression) assign.getExpressions().get(0).getValue();
+ if (logicalExpression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression functionCall = (AbstractFunctionCallExpression) logicalExpression;
+ if (!functionCall.getFunctionIdentifier().equals(BuiltinFunctions.FN_COLLECTION_1.getFunctionIdentifier())) {
+ return false;
+ }
+
+ // Get the string assigned to the collection function.
+ AbstractLogicalOperator op3 = (AbstractLogicalOperator) assign.getInputs().get(0).getValue();
+ if (op3.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ AssignOperator assign2 = (AssignOperator) op3;
+
+ // Check to see if the expression is a constant expression and type string.
+ ILogicalExpression logicalExpression2 = (ILogicalExpression) assign2.getExpressions().get(0).getValue();
+ if (logicalExpression2.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return false;
+ }
+ ConstantExpression constantExpression = (ConstantExpression) logicalExpression2;
+ VXQueryConstantValue constantValue = (VXQueryConstantValue) constantExpression.getValue();
+ if (constantValue.getType() != SequenceType.create(BuiltinTypeRegistry.XS_STRING, Quantifier.QUANT_ONE)) {
+ return false;
+ }
+
+ // Constant value is now in a TaggedValuePointable. Convert the value into a java String.
+ tvp.set(constantValue.getValue(), 0, constantValue.getValue().length);
+ String collectionName = null;
+ if (tvp.getTag() == ValueTag.XS_STRING_TAG) {
+ tvp.getValue(stringp);
+ try {
+ bbis.setByteBuffer(
+ ByteBuffer.wrap(Arrays.copyOfRange(stringp.getByteArray(), stringp.getStartOffset(),
+ stringp.getLength() + stringp.getStartOffset())), 0);
+ collectionName = di.readUTF();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // Build the new operator and update the query plan.
+ List<Object> types = new ArrayList<Object>();
+ types.add(SequenceType.create(AnyItemType.INSTANCE, Quantifier.QUANT_STAR));
+ VXQueryCollectionDataSource ds = new VXQueryCollectionDataSource(collectionName, types.toArray());
+ DataSourceScanOperator opNew = new DataSourceScanOperator(unnest.getVariables(), ds);
+ opNew.getInputs().addAll(assign.getInputs());
+ opRef.setValue(opNew);
+ return true;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/CollectionRewriteRule.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java?rev=1399491&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java Thu Oct 18 00:28:21 2012
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.vxquery.compiler.rewriter.rules.CollectionFileDomain;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+
+public class VXQueryCollectionDataSource implements IDataSource<String> {
+ private final String collectionName;
+
+ private final Object[] types;
+
+ private IDataSourcePropertiesProvider propProvider;
+
+ public VXQueryCollectionDataSource(String file, Object[] types) {
+ this.collectionName = file;
+ this.types = types;
+ final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(new RandomPartitioningProperty(
+ new CollectionFileDomain(collectionName)), new ArrayList<ILocalStructuralProperty>());
+ propProvider = new IDataSourcePropertiesProvider() {
+ @Override
+ public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
+ return vec;
+ }
+ };
+ }
+
+ @Override
+ public String getId() {
+ return collectionName;
+ }
+
+ @Override
+ public Object[] getSchemaTypes() {
+ return types;
+ }
+
+ @Override
+ public IDataSourcePropertiesProvider getPropertiesProvider() {
+ return propProvider;
+ }
+
+ @Override
+ public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList) {
+ }
+
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java?rev=1399491&r1=1399490&r2=1399491&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java Thu Oct 18 00:28:21 2012
@@ -156,6 +156,8 @@ public class XMLQueryCompiler {
SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+ RewriteRuleset.buildXQueryNormalizationRuleCollection()));
defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
RewriteRuleset.buildTypeInferenceRuleCollection()));
defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlFullDfs,