You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2022/12/05 22:34:55 UTC
Change in asterixdb[neo]: [WIP] Extract function calls from record constructors
From Wail Alkowaileet <wa...@gmail.com>:
Wail Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17300 )
Change subject: [WIP] Extract function calls from record constructors
......................................................................
[WIP] Extract function calls from record constructors
Change-Id: I59c524a109ef8e18a11844a96b690eac115b27f0
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractFunctionExpressionsFromRecordConstructorRule.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
5 files changed, 166 insertions(+), 24 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/00/17300/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 5493dd1..9045517 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -42,6 +42,7 @@
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
import org.apache.asterix.optimizer.rules.ExtractBatchableExternalFunctionCallsRule;
import org.apache.asterix.optimizer.rules.ExtractDistinctByExpressionsRule;
+import org.apache.asterix.optimizer.rules.ExtractFunctionExpressionsFromRecordConstructorRule;
import org.apache.asterix.optimizer.rules.ExtractOrderExpressionsRule;
import org.apache.asterix.optimizer.rules.ExtractWindowExpressionsRule;
import org.apache.asterix.optimizer.rules.FeedScanCollectionToUnnest;
@@ -196,6 +197,7 @@
normalization.add(new ExtractDistinctByExpressionsRule());
normalization.add(new ExtractOrderExpressionsRule());
normalization.add(new ExtractWindowExpressionsRule());
+ normalization.add(new ExtractFunctionExpressionsFromRecordConstructorRule());
// IntroduceStaticTypeCastRule should go before
// IntroduceDynamicTypeCastRule to
@@ -238,8 +240,8 @@
condPushDownAndJoinInference.add(new RemoveCartesianProductWithEmptyBranchRule());
condPushDownAndJoinInference.add(new PushMapOperatorDownThroughProductRule());
condPushDownAndJoinInference.add(new PushSubplanWithAggregateDownThroughProductRule());
- condPushDownAndJoinInference
- .add(new AsterixPushMapOperatorThroughUnionRule(LogicalOperatorTag.ASSIGN, LogicalOperatorTag.SELECT));
+ condPushDownAndJoinInference.add(
+ new AsterixPushMapOperatorThroughUnionRule(LogicalOperatorTag.ASSIGN, LogicalOperatorTag.SELECT));
condPushDownAndJoinInference.add(new AsterixExtractFunctionsFromJoinConditionRule());
@@ -366,8 +368,9 @@
physicalRewritesAllLevels.add(new HybridToInMemoryHashJoinRule());
physicalRewritesAllLevels.add(new AddEquivalenceClassForRecordConstructorRule());
physicalRewritesAllLevels.add(new CheckFullParallelSortRule());
- physicalRewritesAllLevels.add(new EnforceStructuralPropertiesRule(BuiltinFunctions.RANGE_MAP,
- BuiltinFunctions.LOCAL_SAMPLING, BuiltinFunctions.NULL_WRITER));
+ physicalRewritesAllLevels.add(
+ new EnforceStructuralPropertiesRule(BuiltinFunctions.RANGE_MAP, BuiltinFunctions.LOCAL_SAMPLING,
+ BuiltinFunctions.NULL_WRITER));
physicalRewritesAllLevels.add(new RemoveSortInFeedIngestionRule());
physicalRewritesAllLevels.add(new RemoveUnnecessarySortMergeExchange());
physicalRewritesAllLevels.add(new PushProjectDownRule());
@@ -408,8 +411,8 @@
List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<>();
prepareForJobGenRewrites.add(new InsertProjectBeforeUnionRule());
prepareForJobGenRewrites.add(new SetAsterixPhysicalOperatorsRule());
- prepareForJobGenRewrites
- .add(new IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
+ prepareForJobGenRewrites.add(
+ new IsolateHyracksOperatorsRule(HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
prepareForJobGenRewrites.add(new FixReplicateOperatorOutputsRule());
prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
// Re-infer all types, so that, e.g., the effect of not-is-null is propagated
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractFunctionExpressionsFromRecordConstructorRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractFunctionExpressionsFromRecordConstructorRule.java
new file mode 100644
index 0000000..68fff46
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ExtractFunctionExpressionsFromRecordConstructorRule.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ExtractFunctionExpressionsFromRecordConstructorRule implements IAlgebraicRewriteRule {
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+
+ AssignOperator assignOp = (AssignOperator) op;
+ boolean changed = false;
+ for (Mutable<ILogicalExpression> exprRef : assignOp.getExpressions()) {
+ if (isObjectConstructor(exprRef)) {
+ changed |= extractObjectConstructorFunctions(context, assignOp, exprRef);
+ }
+ }
+
+ return changed;
+ }
+
+ private boolean isObjectConstructor(Mutable<ILogicalExpression> exprRef) {
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ return BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR.equals(fid)
+ || BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR.equals(fid);
+ }
+
+ private boolean extractObjectConstructorFunctions(IOptimizationContext context, AssignOperator assignOp,
+ Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) exprRef.getValue();
+ boolean changed = false;
+ for (Mutable<ILogicalExpression> argRef : funcExpr.getArguments()) {
+ changed |= extractFunction(context, assignOp, argRef);
+ }
+ return changed;
+ }
+
+ private boolean extractFunction(IOptimizationContext context, AssignOperator assignOp,
+ Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+
+ LogicalVariable newVar = context.newVar();
+ List<LogicalVariable> newAssignVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> newAssignArgs = new ArrayList<>();
+
+ newAssignVars.add(newVar);
+ newAssignArgs.add(new MutableObject<>(expr));
+ AssignOperator newAssign = new AssignOperator(newAssignVars, newAssignArgs);
+
+ // Replace the function with a variable reference expression
+ exprRef.setValue(new VariableReferenceExpression(newVar));
+
+ // Inject newAssign below the assignOp
+ Mutable<ILogicalOperator> assignInput = assignOp.getInputs().get(0);
+ newAssign.getInputs().add(new MutableObject<>(assignInput.getValue()));
+ assignInput.setValue(newAssign);
+
+ context.computeAndSetTypeEnvironmentForOperator(newAssign);
+ return true;
+ }
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
index b9d512b..991c69c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.optimizer.rules;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
@@ -27,6 +30,7 @@
import org.apache.asterix.algebra.base.OperatorAnnotation;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -34,8 +38,6 @@
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.optimizer.base.AnalysisUtil;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -64,7 +66,7 @@
public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
- private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
+ private final ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
@@ -98,13 +100,13 @@
// checking if we can annotate a Selection as using just one field
// access
SelectOperator sigma = (SelectOperator) op1;
- LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
+ List<LogicalVariable> vars = new ArrayList<>();
VariableUtilities.getUsedVariables(sigma, vars);
if (vars.size() == 1) {
// we can annotate Selection
AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
- AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
- if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ ILogicalExpression expr1 = getFirstExpr(assign1);
+ if (FunctionUtil.isFieldAccessFunction(expr1)) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
// f should be a call to a field/data access kind of
// function
@@ -141,7 +143,7 @@
}
// create an assign
LogicalVariable v = context.newVar();
- AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
+ AssignOperator a2 = new AssignOperator(v, new MutableObject<>(f));
a2.setSourceLocation(expr.getSourceLocation());
pushFieldAssign(a2, topOp, context);
context.computeAndSetTypeEnvironmentForOperator(a2);
@@ -151,7 +153,7 @@
LogicalVariable var = ref.getVariableReference();
List<LogicalVariable> keys = context.findPrimaryKey(var);
if (keys != null) {
- List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> tail = new ArrayList<>();
tail.add(v);
FunctionalDependency pk = new FunctionalDependency(keys, tail);
context.addPrimaryKey(pk);
@@ -381,8 +383,9 @@
Integer fieldIndex = (Integer) index;
ARecordType recordType = (ARecordType) typeEnvironment.getType(fce);
String[] closedFieldNames = recordType.getFieldNames();
- return closedFieldNames.length > fieldIndex
- ? findFieldByNameFromRecordConstructor(closedFieldNames[fieldIndex], fce) : null;
+ return closedFieldNames.length > fieldIndex ?
+ findFieldByNameFromRecordConstructor(closedFieldNames[fieldIndex], fce) :
+ null;
}
private final class ExtractFieldLoadExpressionVisitor implements ILogicalExpressionReferenceTransform {
@@ -408,5 +411,4 @@
private static ILogicalExpression getFirstExpr(AssignOperator assign) {
return assign.getExpressions().get(0).getValue();
}
-
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 42a35f3..fc84ed3 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -19,6 +19,8 @@
package org.apache.asterix.lang.common.util;
+import org.apache.commons.lang3.mutable.Mutable;
+
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
@@ -54,10 +56,10 @@
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.TypeSignature;
import org.apache.asterix.om.utils.ConstantExpressionUtil;
-import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -233,18 +235,21 @@
}
public static boolean isBuiltinDatasetFunction(FunctionSignature fs) {
- return Objects.equals(FN_DATASET_DATAVERSE_NAME, fs.getDataverseName())
- && Objects.equals(FN_DATASET_NAME, fs.getName());
+ return Objects.equals(FN_DATASET_DATAVERSE_NAME, fs.getDataverseName()) && Objects.equals(FN_DATASET_NAME,
+ fs.getName());
}
public static Triple<DatasetFullyQualifiedName, Boolean, DatasetFullyQualifiedName> parseDatasetFunctionArguments(
AbstractCallExpression datasetFn) throws CompilationException {
List<Expression> argList = datasetFn.getExprList();
- DatasetFullyQualifiedName datasetOrViewName = parseDatasetFunctionArguments(argList, 0,
- datasetFn.getSourceLocation(), ExpressionUtils::getStringLiteral);
+ DatasetFullyQualifiedName datasetOrViewName =
+ parseDatasetFunctionArguments(argList, 0, datasetFn.getSourceLocation(),
+ ExpressionUtils::getStringLiteral);
boolean isView = argList.size() > 2 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(argList.get(2)));
- DatasetFullyQualifiedName synonymName = argList.size() > 3 ? parseDatasetFunctionArguments(argList, 3,
- datasetFn.getSourceLocation(), ExpressionUtils::getStringLiteral) : null;
+ DatasetFullyQualifiedName synonymName = argList.size() > 3 ?
+ parseDatasetFunctionArguments(argList, 3, datasetFn.getSourceLocation(),
+ ExpressionUtils::getStringLiteral) :
+ null;
return new Triple<>(datasetOrViewName, isView, synonymName);
}
@@ -303,4 +308,16 @@
function.getSignature(), e.getMessage());
}
}
+
+ public static boolean isFieldAccessFunction(ILogicalExpression expression) {
+ if (expression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ return BuiltinFunctions.FIELD_ACCESS_BY_INDEX.equals(fid) || BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid)
+ || BuiltinFunctions.FIELD_ACCESS_NESTED.equals(fid);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
index c0197a4..0828b35 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/resources/log4j2.xml
@@ -36,6 +36,7 @@
<Root level="WARN">
<AppenderRef ref="Console"/>
</Root>
+ <Logger name="org.apache.hyracks.algebricks" level="TRACE"/>
<Logger name="org.apache.hyracks.control.nc.service" level="INFO"/>
<Logger name="org.apache.hyracks.http.server.CLFLogger" level="ACCESS" additivity="false">
<AppenderRef ref="AccessLog"/>
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17300
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: neo
Gerrit-Change-Id: I59c524a109ef8e18a11844a96b690eac115b27f0
Gerrit-Change-Number: 17300
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <wa...@gmail.com>
Gerrit-MessageType: newchange