You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2013/06/20 22:29:20 UTC
svn commit: r1495166 - in /incubator/vxquery/trunk/vxquery:
vxquery-cli/src/main/java/org/apache/vxquery/cli/
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/
vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/
vxquer...
Author: prestonc
Date: Thu Jun 20 20:29:19 2013
New Revision: 1495166
URL: http://svn.apache.org/r1495166
Log:
Updating the system to use Hyracks 0.2.6 and bringing the rest of the rewrite rules. The system still has an issue with 0.2.6. A query does not quit on its own. I am committing this patch so more people can look at this and help fix this issue.
Added:
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java (with props)
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java (with props)
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java (with props)
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO (with props)
Modified:
incubator/vxquery/trunk/vxquery/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ConsolidateAssignAggregateRule.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/EliminateSubplanForSinglePathsRule.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/RemoveUnusedSortDistinctNodesRule.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
incubator/vxquery/trunk/vxquery/vxquery-parent/pom.xml
incubator/vxquery/trunk/vxquery/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunnerFactory.java
Modified: incubator/vxquery/trunk/vxquery/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java Thu Jun 20 20:29:19 2013
@@ -170,7 +170,7 @@ public class VXQuery {
}
private void runInProcess(JobSpecification spec, File result) throws Exception {
- JobId jobId = hcc.startJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
hcc.waitForCompletion(jobId);
dumpOutputFiles(result);
}
@@ -212,6 +212,7 @@ public class VXQuery {
ncConfig1.ccPort = 39001;
ncConfig1.clusterNetIPAddress = "127.0.0.1";
ncConfig1.dataIPAddress = "127.0.0.1";
+ ncConfig1.datasetIPAddress = "127.0.0.1";
ncConfig1.nodeId = "nc1";
nc1 = new NodeControllerService(ncConfig1);
nc1.start();
@@ -221,12 +222,12 @@ public class VXQuery {
ncConfig2.ccPort = 39001;
ncConfig2.clusterNetIPAddress = "127.0.0.1";
ncConfig2.dataIPAddress = "127.0.0.1";
+ ncConfig2.datasetIPAddress = "127.0.0.1";
ncConfig2.nodeId = "nc2";
nc2 = new NodeControllerService(ncConfig2);
nc2.start();
hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
- hcc.createApplication("test", null);
}
public void stopLocalHyracks() throws Exception {
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/RewriteRuleset.java Thu Jun 20 20:29:19 2013
@@ -20,11 +20,17 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.vxquery.compiler.rewriter.rules.ConsolidateAssignAggregateRule;
+import org.apache.vxquery.compiler.rewriter.rules.ConvertAssignSortDistinctNodesToOperatorsRule;
+import org.apache.vxquery.compiler.rewriter.rules.ConvertAssignToAggregateRule;
+import org.apache.vxquery.compiler.rewriter.rules.EliminateSubplanForSinglePathsRule;
import org.apache.vxquery.compiler.rewriter.rules.EliminateUnnestAggregateSubplanRule;
+import org.apache.vxquery.compiler.rewriter.rules.InlineReferenceVariablePolicy;
import org.apache.vxquery.compiler.rewriter.rules.IntroduceCollectionRule;
import org.apache.vxquery.compiler.rewriter.rules.IntroduceTwoStepAggregateRule;
import org.apache.vxquery.compiler.rewriter.rules.RemoveUnusedSortDistinctNodesRule;
import org.apache.vxquery.compiler.rewriter.rules.SetCollectionDataSourceRule;
+import org.apache.vxquery.compiler.rewriter.rules.SetVariableIdContextRule;
+import org.apache.vxquery.compiler.rewriter.rules.temporary.InlineVariablesRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
@@ -39,7 +45,6 @@ import edu.uci.ics.hyracks.algebricks.re
import edu.uci.ics.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.InferTypesRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineAssignIntoAggregateRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.IntroduceAggregateCombinerRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.IntroduceGroupByCombinerRule;
import edu.uci.ics.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule;
@@ -57,11 +62,25 @@ import edu.uci.ics.hyracks.algebricks.re
public class RewriteRuleset {
public final static List<IAlgebraicRewriteRule> buildXQueryNormalizationRuleCollection() {
List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
+ normalization.add(new SetVariableIdContextRule());
+
normalization.add(new RemoveUnusedSortDistinctNodesRule());
- normalization.add(new InlineVariablesRule());
+ normalization.add(new InlineVariablesRule(new InlineReferenceVariablePolicy()));
+ normalization.add(new RemoveUnusedAssignAndAggregateRule());
+ // TODO Fix the group by operator before putting back in the rule set.
+// normalization.add(new ConvertAssignSortDistinctNodesToOperatorsRule());
+
normalization.add(new ConsolidateAssignAggregateRule());
+ normalization.add(new InlineVariablesRule(new InlineReferenceVariablePolicy()));
normalization.add(new RemoveUnusedAssignAndAggregateRule());
+
+ normalization.add(new ConvertAssignToAggregateRule());
+
normalization.add(new EliminateUnnestAggregateSubplanRule());
+ normalization.add(new InlineVariablesRule(new InlineReferenceVariablePolicy()));
+ normalization.add(new RemoveUnusedAssignAndAggregateRule());
+
+ normalization.add(new EliminateSubplanForSinglePathsRule());
normalization.add(new SetCollectionDataSourceRule());
normalization.add(new IntroduceCollectionRule());
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/VXQueryOptimizationContext.java Thu Jun 20 20:29:19 2013
@@ -38,13 +38,12 @@ public class VXQueryOptimizationContext
private final Map<String, VXQueryCollectionDataSource> dataSourceScanMap = new HashMap<String, VXQueryCollectionDataSource>();
- public VXQueryOptimizationContext(int varCounter, int frameSize,
- IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ public VXQueryOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
- IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
- PhysicalOptimizationConfig physicalOptimizationConfig) {
- super(varCounter, frameSize, expressionEvalSizeComputer, mergeAggregationExpressionFactory,
- expressionTypeComputer, nullableTypeComputer, physicalOptimizationConfig);
+ IExpressionTypeComputer expressionTypeComputer,
+ INullableTypeComputer nullableTypeComputer, PhysicalOptimizationConfig physicalOptimizationConfig) {
+ super(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
+ nullableTypeComputer, physicalOptimizationConfig);
}
public VXQueryCollectionDataSource getCollectionDataSourceMap(String collectionName) {
@@ -54,11 +53,11 @@ public class VXQueryOptimizationContext
return null;
}
}
-
+
public int getCollectionDataSourceMapSize() {
return dataSourceScanMap.size();
}
-
+
public void putCollectionDataSourceMap(String collectionName, VXQueryCollectionDataSource ds) {
this.dataSourceScanMap.put(collectionName, ds);
}
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ConsolidateAssignAggregateRule.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ConsolidateAssignAggregateRule.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ConsolidateAssignAggregateRule.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/ConsolidateAssignAggregateRule.java Thu Jun 20 20:29:19 2013
@@ -131,7 +131,8 @@ public class ConsolidateAssignAggregateR
trueAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(aggInput));
// Set partitioning variable.
- aggregate.setPartitioningVariable(trueVar);
+ // TODO Review why this is not valid in 0.2.6
+// aggregate.setPartitioningVariable(trueVar);
return true;
}
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/EliminateSubplanForSinglePathsRule.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/EliminateSubplanForSinglePathsRule.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/EliminateSubplanForSinglePathsRule.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/EliminateSubplanForSinglePathsRule.java Thu Jun 20 20:29:19 2013
@@ -141,7 +141,6 @@ public class EliminateSubplanForSinglePa
// The following operators' analysis has not yet been implemented.
case CLUSTER:
- case DIE:
case DISTINCT:
case EXTENSION_OPERATOR:
case INDEX_INSERT_DELETE:
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java?rev=1495166&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java Thu Jun 20 20:29:19 2013
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.compiler.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.vxquery.compiler.rewriter.rules.temporary.InlineVariablesRule;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+
+/**
+ * Where assign operators only assign a new variable ID for a reference expression,
+ * all references are updated to the first variable ID.
+ */
+public class InlineReferenceVariablePolicy implements InlineVariablesRule.IInlineVariablePolicy {
+
+ @Override
+ public boolean addExpressionToInlineMap(ILogicalExpression expr, Set<FunctionIdentifier> doNotInlineFuncs) {
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public List<Mutable<ILogicalOperator>> descendIntoNextOperator(AbstractLogicalOperator op) {
+ List<Mutable<ILogicalOperator>> descendOp = new ArrayList<Mutable<ILogicalOperator>>();
+ // Descend into nested plans removing projects on the way.
+ if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+ SubplanOperator subplan = (SubplanOperator) op;
+ for (ILogicalPlan nestedOpRef : subplan.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> rootOpRef : nestedOpRef.getRoots()) {
+ descendOp.add(rootOpRef);
+ }
+ }
+ }
+ // Descend into children removing projects on the way.
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ descendOp.add(inputOpRef);
+ }
+ return descendOp;
+ }
+
+ @Override
+ public boolean transformOperator(AbstractLogicalOperator op) {
+ return true;
+ }
+
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/InlineReferenceVariablePolicy.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/RemoveUnusedSortDistinctNodesRule.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/RemoveUnusedSortDistinctNodesRule.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/RemoveUnusedSortDistinctNodesRule.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/RemoveUnusedSortDistinctNodesRule.java Thu Jun 20 20:29:19 2013
@@ -332,7 +332,6 @@ public class RemoveUnusedSortDistinctNod
// The following operators' analysis has not yet been implemented.
case CLUSTER:
- case DIE:
case DISTINCT:
case EXTENSION_OPERATOR:
case INDEX_INSERT_DELETE:
@@ -441,7 +440,6 @@ public class RemoveUnusedSortDistinctNod
// The following operators' analysis has not yet been implemented.
case CLUSTER:
- case DIE:
case DISTINCT:
case EXTENSION_OPERATOR:
case GROUP:
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java?rev=1495166&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java Thu Jun 20 20:29:19 2013
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.compiler.rewriter.rules.temporary;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+
+/**
+ * Where assign operators only assign a new variable ID for a reference expression,
+ * all references are updated to the first variable ID.
+ */
+public class InlineVariablePolicy implements InlineVariablesRule.IInlineVariablePolicy {
+
+ @Override
+ public boolean addExpressionToInlineMap(ILogicalExpression expr, Set<FunctionIdentifier> doNotInlineFuncs) {
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ if (doNotInlineFuncs.contains(funcExpr.getFunctionIdentifier())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public List<Mutable<ILogicalOperator>> descendIntoNextOperator(AbstractLogicalOperator op) {
+ List<Mutable<ILogicalOperator>> descendOp = new ArrayList<Mutable<ILogicalOperator>>();
+ // Descend into children removing projects on the way.
+ for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+ descendOp.add(inputOpRef);
+ }
+ return descendOp;
+ }
+
+ @Override
+ public boolean transformOperator(AbstractLogicalOperator op) {
+ // Only inline variables in operators that can deal with arbitrary expressions.
+ if (!op.requiresVariableReferenceExpressions()) {
+ return true;
+ }
+ return false;
+ }
+
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablePolicy.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java?rev=1495166&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java Thu Jun 20 20:29:19 2013
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.vxquery.compiler.rewriter.rules.temporary;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Replaces variable reference expressions with their assigned function-call expression where applicable
+ * (some variables are generated by datasources).
+ * Inlining variables may enable other optimizations by allowing selects and assigns to be moved
+ * (e.g., a select may be pushed into a join to enable an efficient physical join operator).
+ *
+ * Preconditions/Assumptions:
+ * Assumes no projects are in the plan. Only inlines variables whose assigned expression is a function call
+ * (i.e., this rule ignores right-hand side constants and other variable references expressions
+ *
+ * Postconditions/Examples:
+ * All qualifying variables have been inlined.
+ *
+ * Example (simplified):
+ *
+ * Before plan:
+ * select <- [$$1 < $$2 + $$0]
+ * assign [$$2] <- [funcZ() + $$0]
+ * assign [$$0, $$1] <- [funcX(), funcY()]
+ *
+ * After plan:
+ * select <- [funcY() < funcZ() + funcX() + funcX()]
+ * assign [$$2] <- [funcZ() + funcX()]
+ * assign [$$0, $$1] <- [funcX(), funcY()]
+ */
+public class InlineVariablesRule implements IAlgebraicRewriteRule {
+
+ // Map of variables that could be replaced by their producing expression.
+ // Populated during the top-down sweep of the plan.
+ protected Map<LogicalVariable, ILogicalExpression> varAssignRhs = new HashMap<LogicalVariable, ILogicalExpression>();
+
+ // Visitor for replacing variable reference expressions with their originating expression.
+ protected InlineVariablesVisitor inlineVisitor = new InlineVariablesVisitor(varAssignRhs);
+
+ // Set of FunctionIdentifiers that we should not inline.
+ protected Set<FunctionIdentifier> doNotInlineFuncs = new HashSet<FunctionIdentifier>();
+
+ protected boolean hasRun = false;
+
+ protected IInlineVariablePolicy policy;
+
+ public InlineVariablesRule() {
+ this.policy = new InlineVariablePolicy();
+ }
+
+ public InlineVariablesRule(IInlineVariablePolicy policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ if (hasRun) {
+ return false;
+ }
+ if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ return false;
+ }
+ prepare(context);
+ boolean modified = inlineVariables(opRef, context);
+ if (performFinalAction()) {
+ modified = true;
+ }
+ hasRun = true;
+ return modified;
+ }
+
+ protected void prepare(IOptimizationContext context) {
+ varAssignRhs.clear();
+ inlineVisitor.setContext(context);
+ }
+
+ protected boolean performBottomUpAction(AbstractLogicalOperator op) throws AlgebricksException {
+ if (policy.transformOperator(op)) {
+ inlineVisitor.setOperator(op);
+ return op.acceptExpressionTransform(inlineVisitor);
+ }
+ return false;
+ }
+
+ protected boolean performFinalAction() throws AlgebricksException {
+ return false;
+ }
+
+ protected boolean inlineVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ // Update mapping from variables to expressions during top-down traversal.
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOp = (AssignOperator) op;
+ List<LogicalVariable> vars = assignOp.getVariables();
+ List<Mutable<ILogicalExpression>> exprs = assignOp.getExpressions();
+ for (int i = 0; i < vars.size(); i++) {
+ ILogicalExpression expr = exprs.get(i).getValue();
+ if (policy.addExpressionToInlineMap(expr, doNotInlineFuncs)) {
+ varAssignRhs.put(vars.get(i), exprs.get(i).getValue());
+ }
+ }
+ }
+
+ boolean modified = false;
+ // Follow all operators from this operator.
+ for (Mutable<ILogicalOperator> inputOpRef : policy.descendIntoNextOperator(op)) {
+ if (inlineVariables(inputOpRef, context)) {
+ modified = true;
+ }
+ }
+
+ if (performBottomUpAction(op)) {
+ modified = true;
+ }
+
+ if (modified) {
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ context.addToDontApplySet(this, op);
+ // Re-enable rules that we may have already tried. They could be applicable now after inlining.
+ context.removeFromAlreadyCompared(opRef.getValue());
+ }
+
+ return modified;
+ }
+
+ protected class InlineVariablesVisitor implements ILogicalExpressionReferenceTransform {
+
+ private final Map<LogicalVariable, ILogicalExpression> varAssignRhs;
+ private final Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
+ private final List<LogicalVariable> rhsUsedVars = new ArrayList<LogicalVariable>();
+ private ILogicalOperator op;
+ private IOptimizationContext context;
+ // If set, only replace this variable reference.
+ private LogicalVariable targetVar;
+
+ public InlineVariablesVisitor(Map<LogicalVariable, ILogicalExpression> varAssignRhs) {
+ this.varAssignRhs = varAssignRhs;
+ }
+
+ public void setTargetVariable(LogicalVariable targetVar) {
+ this.targetVar = targetVar;
+ }
+
+ public void setContext(IOptimizationContext context) {
+ this.context = context;
+ }
+
+ public void setOperator(ILogicalOperator op) throws AlgebricksException {
+ this.op = op;
+ liveVars.clear();
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+ ILogicalExpression e = exprRef.getValue();
+ switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+ case VARIABLE: {
+ LogicalVariable var = ((VariableReferenceExpression) e).getVariableReference();
+ // Restrict replacement to targetVar if it has been set.
+ if (targetVar != null && var != targetVar) {
+ return false;
+ }
+ // Make sure has not been excluded from inlining.
+ if (context.shouldNotBeInlined(var)) {
+ return false;
+ }
+ ILogicalExpression rhs = varAssignRhs.get(var);
+ if (rhs == null) {
+ // Variable was not produced by an assign.
+ return false;
+ }
+
+ // Make sure used variables from rhs are live.
+ if (liveVars.isEmpty()) {
+ VariableUtilities.getLiveVariables(op, liveVars);
+ }
+ rhsUsedVars.clear();
+ rhs.getUsedVariables(rhsUsedVars);
+ for (LogicalVariable rhsUsedVar : rhsUsedVars) {
+ if (!liveVars.contains(rhsUsedVar)) {
+ return false;
+ }
+ }
+
+ // Replace variable reference with a clone of the rhs expr.
+ exprRef.setValue(rhs.cloneExpression());
+ return true;
+ }
+ case FUNCTION_CALL: {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
+ boolean modified = false;
+ for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
+ if (transform(arg)) {
+ modified = true;
+ }
+ }
+ return modified;
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+ }
+
+ public static interface IInlineVariablePolicy {
+
+ public boolean addExpressionToInlineMap(ILogicalExpression expr, Set<FunctionIdentifier> doNotInlineFuncs);
+
+ public List<Mutable<ILogicalOperator>> descendIntoNextOperator(AbstractLogicalOperator op);
+
+ public boolean transformOperator(AbstractLogicalOperator op);
+
+ }
+
+}
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/InlineVariablesRule.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO?rev=1495166&view=auto
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO (added)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO Thu Jun 20 20:29:19 2013
@@ -0,0 +1 @@
+Update the rewriter ruleset to use the new hyracks InlineVariableRule and remove this folder and all files after hyracks issue #99 has been resolved.
\ No newline at end of file
Propchange: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/compiler/rewriter/rules/temporary/TODO
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryMetadataProvider.java Thu Jun 20 20:29:19 2013
@@ -52,8 +52,8 @@ public class VXQueryMetadataProvider imp
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec)
- throws AlgebricksException {
+ IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
+ JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
VXQueryCollectionDataSource ds = (VXQueryCollectionDataSource) dataSource;
RecordDescriptor rDesc = new RecordDescriptor(new ISerializerDeserializer[opSchema.getSize()]);
IOperatorDescriptor scanner = new VXQueryCollectionOperatorDescriptor(jobSpec, ds.getId(),
@@ -96,15 +96,17 @@ public class VXQueryMetadataProvider imp
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
- IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
- RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException {
return null;
}
@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<String> dataSource,
- IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
- RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec)
+ throws AlgebricksException {
return null;
}
@@ -136,4 +138,12 @@ public class VXQueryMetadataProvider imp
public IFunctionInfo lookupFunction(FunctionIdentifier fid) {
return null;
}
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
+ int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
+ JobSpecification spec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
\ No newline at end of file
Modified: incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-core/src/main/java/org/apache/vxquery/xmlquery/query/XMLQueryCompiler.java Thu Jun 20 20:29:19 2013
@@ -86,12 +86,12 @@ public class XMLQueryCompiler {
HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(
new IOptimizationContextFactory() {
@Override
- public IOptimizationContext createOptimizationContext(int varCounter, int frameSize,
+ public IOptimizationContext createOptimizationContext(int varCounter,
IExpressionEvalSizeComputer expressionEvalSizeComputer,
IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
PhysicalOptimizationConfig physicalOptimizationConfig) {
- return new VXQueryOptimizationContext(varCounter, frameSize, expressionEvalSizeComputer,
+ return new VXQueryOptimizationContext(varCounter, expressionEvalSizeComputer,
mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer,
physicalOptimizationConfig);
}
@@ -156,7 +156,7 @@ public class XMLQueryCompiler {
listener.notifyOptimizedResult(module);
JobSpecification jobSpec;
try {
- jobSpec = compiler.createJob(null);
+ jobSpec = compiler.createJob(null, null);
} catch (AlgebricksException e) {
throw new SystemException(ErrorCode.SYSE0001, e);
}
Modified: incubator/vxquery/trunk/vxquery/vxquery-parent/pom.xml
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-parent/pom.xml?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-parent/pom.xml (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-parent/pom.xml Thu Jun 20 20:29:19 2013
@@ -231,7 +231,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <hyracks.version>0.2.3-SNAPSHOT</hyracks.version>
+ <hyracks.version>0.2.6</hyracks.version>
</properties>
</project>
Modified: incubator/vxquery/trunk/vxquery/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunnerFactory.java
URL: http://svn.apache.org/viewvc/incubator/vxquery/trunk/vxquery/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunnerFactory.java?rev=1495166&r1=1495165&r2=1495166&view=diff
==============================================================================
--- incubator/vxquery/trunk/vxquery/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunnerFactory.java (original)
+++ incubator/vxquery/trunk/vxquery/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/TestRunnerFactory.java Thu Jun 20 20:29:19 2013
@@ -83,6 +83,7 @@ public class TestRunnerFactory {
ncConfig1.ccPort = 39001;
ncConfig1.clusterNetIPAddress = "127.0.0.1";
ncConfig1.dataIPAddress = "127.0.0.1";
+ ncConfig1.datasetIPAddress = "127.0.0.1";
ncConfig1.nodeId = "nc1";
nc1 = new NodeControllerService(ncConfig1);
nc1.start();
@@ -92,12 +93,12 @@ public class TestRunnerFactory {
ncConfig2.ccPort = 39001;
ncConfig2.clusterNetIPAddress = "127.0.0.1";
ncConfig2.dataIPAddress = "127.0.0.1";
+ ncConfig2.datasetIPAddress = "127.0.0.1";
ncConfig2.nodeId = "nc2";
nc2 = new NodeControllerService(ncConfig2);
nc2.start();
hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
- hcc.createApplication("test", null);
}
public void registerReporter(ResultReporter reporter) {
@@ -129,7 +130,7 @@ public class TestRunnerFactory {
spec.setGlobalJobDataFactory(new VXQueryGlobalDataFactory(dCtx.createFactory()));
spec.setMaxReattempts(0);
- JobId jobId = hcc.startJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ JobId jobId = hcc.startJob(spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
hcc.waitForCompletion(jobId);
res.result = FileUtils.readFileToString(tempFile, "UTF-8").trim();
} catch (HyracksException e) {