You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/09/26 12:48:58 UTC
[L] Change in asterixdb[master]: Merge branch 'gerrit/morpheus' into 'master'
From Michael Blow <mb...@apache.org>:
Michael Blow has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21905?usp=email )
Change subject: Merge branch 'gerrit/morpheus' into 'master'
......................................................................
Merge branch 'gerrit/morpheus' into 'master'
* [NO ISSUE][MISC] Advance the AWS SDK to 2.55.5 (MB-73268)
* [NO ISSUE][MISC] Advance the GCS SDK to 2.74.0 (MB-73268)
* [ASTERIXDB-3899][COMP] Do not constant-fold median (MB-74216)
The median guard goes in ConstantFoldingVisitor, where canConstantFold
lives on this branch, and hyracks-fullstack takes the same SDK versions
per its keep-in-sync notes.
Generated-By: Claude Opus 5.5 <no...@anthropic.com>
Ext-ref: MB-73268,MB-74216
Change-Id: I740c30cb5c70937e675c7acac07ed5bde5d73812
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/visitor/ConstantFoldingVisitor.java
M asterixdb/pom.xml
M hyracks-fullstack/pom.xml
5 files changed, 18 insertions(+), 349 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/05/21905/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index a0ef247..343ff5d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -51,338 +51,4 @@
cfv.reset(context);
return op.acceptExpressionTransform(cfv);
}
-<<<<<<< HEAD (f7f02c [ASTERIXDB-3830][COMP] Enable hash-based COUNT(DISTINCT) by )
-=======
-
- private class ConstantFoldingVisitor implements ILogicalExpressionVisitor<Pair<Boolean, ILogicalExpression>, Void>,
- ILogicalExpressionReferenceTransform, IEvaluatorContext {
-
- private final IPointable p = VoidPointable.FACTORY.createPointable();
- private final ByteBufferInputStream bbis = new ByteBufferInputStream();
- private final DataInputStream dis = new DataInputStream(bbis);
- private final WarningCollector warningCollector = new WarningCollector();
- private IOptimizationContext optContext;
- private IServiceContext serviceContext;
-
- private void reset(IOptimizationContext context) {
- optContext = context;
- serviceContext =
- ((MetadataProvider) context.getMetadataProvider()).getApplicationContext().getServiceContext();
- }
-
- @Override
- public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
- AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
- Pair<Boolean, ILogicalExpression> newExpression = expr.accept(this, null);
- if (newExpression.first) {
- exprRef.setValue(newExpression.second);
- }
- return newExpression.first;
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitConstantExpression(ConstantExpression expr, Void arg) {
- return new Pair<>(false, expr);
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitVariableReferenceExpression(VariableReferenceExpression expr,
- Void arg) {
- return new Pair<>(false, expr);
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr,
- Void arg) throws AlgebricksException {
- boolean changed = constantFoldArgs(expr, arg);
- List<Mutable<ILogicalExpression>> argList = expr.getArguments();
- int argConstantCount = countConstantArgs(argList);
- FunctionIdentifier fid = expr.getFunctionIdentifier();
- if (argConstantCount != argList.size()) {
- if (argConstantCount > 0 && (BuiltinFunctions.OR.equals(fid) || BuiltinFunctions.AND.equals(fid))) {
- if (foldOrAndArgs(expr)) {
- ILogicalExpression changedExpr =
- expr.getArguments().size() == 1 ? expr.getArguments().get(0).getValue() : expr;
- return new Pair<>(true, changedExpr);
- }
- }
- return new Pair<>(changed, expr);
- }
-
- if (!expr.isFunctional() || !canConstantFold(expr)) {
- return new Pair<>(changed, expr);
- }
-
- try {
- if (BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid)) {
- IAType argType = (IAType) _emptyTypeEnv.getType(expr.getArguments().get(0).getValue());
- if (argType.getTypeTag() == ATypeTag.OBJECT) {
- ARecordType rt = (ARecordType) argType;
- String str = ConstantExpressionUtil.getStringConstant(expr.getArguments().get(1).getValue());
- int k = rt.getFieldIndex(str);
- if (k >= 0) {
- // wait for the ByNameToByIndex rule to apply
- return new Pair<>(changed, expr);
- }
- }
- }
- IAObject c = FUNC_ID_TO_CONSTANT.get(fid);
- if (c != null) {
- ConstantExpression constantExpression = new ConstantExpression(new AsterixConstantValue(c));
- constantExpression.setSourceLocation(expr.getSourceLocation());
- return new Pair<>(true, constantExpression);
- }
-
- IScalarEvaluatorFactory fact = jobGenCtx.getExpressionRuntimeProvider().createEvaluatorFactory(expr,
- _emptyTypeEnv, _emptySchemas, jobGenCtx);
-
- warningCollector.clear();
- IScalarEvaluator eval = fact.createScalarEvaluator(this);
- eval.evaluate(null, p);
- IAType returnType = (IAType) _emptyTypeEnv.getType(expr);
- ATypeTag runtimeType = PointableHelper.getTypeTag(p);
- if (runtimeType.isDerivedType()) {
- returnType = TypeComputeUtils.getActualType(returnType);
- } else {
- returnType = TypeTagUtil.getBuiltinTypeByTag(runtimeType);
- }
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer serde =
- jobGenCtx.getSerializerDeserializerProvider().getSerializerDeserializer(returnType);
- bbis.setByteBuffer(ByteBuffer.wrap(p.getByteArray(), p.getStartOffset(), p.getLength()), 0);
- IAObject o = (IAObject) serde.deserialize(dis);
- warningCollector.getWarnings(optContext.getWarningCollector());
- ConstantExpression constantExpression = new ConstantExpression(new AsterixConstantValue(o));
- constantExpression.setSourceLocation(expr.getSourceLocation());
- return new Pair<>(true, constantExpression);
- } catch (HyracksDataException | AlgebricksException e) {
- if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
- AlgebricksConfig.ALGEBRICKS_LOGGER.trace("Exception caught at constant folding: " + e, e);
- }
- return new Pair<>(false, null);
- }
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitAggregateFunctionCallExpression(
- AggregateFunctionCallExpression expr, Void arg) throws AlgebricksException {
- boolean changed = constantFoldArgs(expr, arg);
- return new Pair<>(changed, expr);
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitStatefulFunctionCallExpression(
- StatefulFunctionCallExpression expr, Void arg) throws AlgebricksException {
- boolean changed = constantFoldArgs(expr, arg);
- return new Pair<>(changed, expr);
- }
-
- @Override
- public Pair<Boolean, ILogicalExpression> visitUnnestingFunctionCallExpression(
- UnnestingFunctionCallExpression expr, Void arg) throws AlgebricksException {
- boolean changed = constantFoldArgs(expr, arg);
- return new Pair<>(changed, expr);
- }
-
- private boolean constantFoldArgs(AbstractFunctionCallExpression expr, Void arg) throws AlgebricksException {
- return expr.getFunctionIdentifier().equals(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)
- ? foldRecordArgs(expr, arg) : foldFunctionArgs(expr, arg);
- }
-
- private boolean foldFunctionArgs(AbstractFunctionCallExpression expr, Void arg) throws AlgebricksException {
- boolean changed = false;
- for (Mutable<ILogicalExpression> exprArgRef : expr.getArguments()) {
- changed |= foldArg(exprArgRef, arg);
- }
- return changed;
- }
-
- private boolean foldRecordArgs(AbstractFunctionCallExpression expr, Void arg) throws AlgebricksException {
- if (expr.getArguments().size() % 2 != 0) {
- String functionName = expr.getFunctionIdentifier().getName();
- throw CompilationException.create(ErrorCode.COMPILATION_INVALID_NUM_OF_ARGS, expr.getSourceLocation(),
- functionName);
- }
- boolean changed = false;
- Iterator<Mutable<ILogicalExpression>> iterator = expr.getArguments().iterator();
- int fieldNameIdx = 0;
- while (iterator.hasNext()) {
- Mutable<ILogicalExpression> fieldNameExprRef = iterator.next();
- Pair<Boolean, ILogicalExpression> fieldNameExpr = fieldNameExprRef.getValue().accept(this, arg);
- boolean isDuplicate = false;
- if (fieldNameExpr.first) {
- String fieldName = ConstantExpressionUtil.getStringConstant(fieldNameExpr.second);
- if (fieldName != null) {
- isDuplicate = isDuplicateField(fieldName, fieldNameIdx, expr.getArguments());
- }
- if (isDuplicate) {
- IWarningCollector warningCollector = optContext.getWarningCollector();
- if (warningCollector.shouldWarn()) {
- warningCollector.warn(Warning.of(fieldNameExpr.second.getSourceLocation(),
- ErrorCode.COMPILATION_DUPLICATE_FIELD_NAME, LogRedactionUtil.userData(fieldName)));
- }
- iterator.remove();
- iterator.next();
- iterator.remove();
- } else {
- fieldNameExprRef.setValue(fieldNameExpr.second);
- }
- changed = true;
- }
- if (!isDuplicate) {
- Mutable<ILogicalExpression> fieldValue = iterator.next();
- changed |= foldArg(fieldValue, arg);
- fieldNameIdx += 2;
- }
- }
- return changed;
- }
-
- private boolean isDuplicateField(String fName, int fIdx, List<Mutable<ILogicalExpression>> args) {
- for (int i = 0, size = args.size(); i < size; i += 2) {
- if (i != fIdx && fName.equals(ConstantExpressionUtil.getStringConstant(args.get(i).getValue()))) {
- return true;
- }
- }
- return false;
- }
-
- private boolean foldArg(Mutable<ILogicalExpression> exprArgRef, Void arg) throws AlgebricksException {
- Pair<Boolean, ILogicalExpression> newExpr = exprArgRef.getValue().accept(this, arg);
- if (newExpr.first) {
- exprArgRef.setValue(newExpr.second);
- return true;
- }
- return false;
- }
-
- private int countConstantArgs(List<Mutable<ILogicalExpression>> argList) {
- int n = 0;
- for (Mutable<ILogicalExpression> r : argList) {
- if (r.getValue().getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- n++;
- }
- }
- return n;
- }
-
- private boolean canConstantFold(ScalarFunctionCallExpression function) throws AlgebricksException {
- // skip external functions because they're not available at compile time (on CC)
- IFunctionInfo fi = function.getFunctionInfo();
- if (fi.isExternal()) {
- return false;
- }
- // median sorts through run files, which need the task context that is null at compile time
- FunctionIdentifier fid = function.getFunctionIdentifier();
- if (fid.equals(BuiltinFunctions.SCALAR_SQL_MEDIAN) || fid.equals(BuiltinFunctions.SCALAR_MEDIAN)) {
- return false;
- }
- // skip all functions that would produce records/arrays/multisets (derived types) in their open format
- // this is because constant folding them will make them closed (currently)
- if (fid.equals(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
- return false;
- }
- IAType returnType = (IAType) _emptyTypeEnv.getType(function);
- return canConstantFoldType(returnType);
- }
-
- private boolean canConstantFoldType(IAType returnType) {
- ATypeTag tag = returnType.getTypeTag();
- if (tag == ATypeTag.ANY) {
- // if the function is to return a record (or derived data), that record would (should) be an open record
- return false;
- } else if (tag == ATypeTag.OBJECT) {
- ARecordType recordType = (ARecordType) returnType;
- if (recordType.isOpen()) {
- return false;
- }
- IAType[] fieldTypes = recordType.getFieldTypes();
- for (int i = 0; i < fieldTypes.length; i++) {
- if (!canConstantFoldType(fieldTypes[i])) {
- return false;
- }
- }
- } else if (tag.isListType()) {
- AbstractCollectionType listType = (AbstractCollectionType) returnType;
- return canConstantFoldType(listType.getItemType());
- } else if (tag == ATypeTag.UNION) {
- return canConstantFoldType(((AUnionType) returnType).getActualType());
- }
- return true;
- }
-
- private boolean foldOrAndArgs(ScalarFunctionCallExpression expr) {
- // or(true,x,y) -> true; or(false,x,y) -> or(x,y)
- boolean changed = false;
- List<Mutable<ILogicalExpression>> argList = expr.getArguments();
- Iterator<Mutable<ILogicalExpression>> argIter = argList.iterator();
- Mutable<ILogicalExpression> argFalse = null;
- while (argIter.hasNext()) {
- Mutable<ILogicalExpression> argExprRef = argIter.next();
- ILogicalExpression argExpr = argExprRef.getValue();
- if (argExpr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- continue;
- }
-
- ConstantExpression cExpr = (ConstantExpression) argExpr;
- IAlgebricksConstantValue cValue = cExpr.getValue();
- FunctionIdentifier fid = expr.getFunctionIdentifier();
-
- if (replaceAndReturn(cValue, fid)) {
- // or(true,x,y) -> true;
- // and(false, x, y) -> false
- argList.clear();
- argList.add(argExprRef);
- return true;
- } else if (removeAndContinue(cValue, fid)) {
- // or(false, x, y) -> or(x, y)
- // and(true, x, y) -> and(x, y)
- // remove 'false' (or 'true') from arg list, but save the expression.
- argFalse = argExprRef;
- argIter.remove();
- changed = true;
- }
- }
- if (argList.isEmpty() && argFalse != null) {
- argList.add(argFalse);
- }
- return changed;
- }
-
- private boolean replaceAndReturn(IAlgebricksConstantValue cValue, FunctionIdentifier fid) {
- if (BuiltinFunctions.OR.equals(fid)) {
- return cValue.isTrue();
- } else {
- // BuiltinFunctions.AND
- return cValue.isFalse();
- }
- }
-
- private boolean removeAndContinue(IAlgebricksConstantValue cValue, FunctionIdentifier fid) {
- if (BuiltinFunctions.OR.equals(fid)) {
- return cValue.isFalse();
- } else {
- // BuiltinFunctions.AND
- return cValue.isTrue();
- }
- }
-
- // IEvaluatorContext
-
- @Override
- public IServiceContext getServiceContext() {
- return serviceContext;
- }
-
- @Override
- public IHyracksTaskContext getTaskContext() {
- return null;
- }
-
- @Override
- public IWarningCollector getWarningCollector() {
- return warningCollector;
- }
- }
->>>>>>> BRANCH (6175d4 Merge branch 'gerrit/trinity' into 'gerrit/morpheus')
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/visitor/ConstantFoldingVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/visitor/ConstantFoldingVisitor.java
index 148e761..67abddf 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/visitor/ConstantFoldingVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/visitor/ConstantFoldingVisitor.java
@@ -378,10 +378,15 @@
if (fi.isExternal()) {
return false;
}
+ // median sorts through run files, which need the task context that is null at compile time
+ FunctionIdentifier fid = function.getFunctionIdentifier();
+ if (fid.equals(BuiltinFunctions.SCALAR_SQL_MEDIAN) || fid.equals(BuiltinFunctions.SCALAR_MEDIAN)) {
+ return false;
+ }
IAType returnType = (IAType) _emptyTypeEnv.getType(function);
// skip all functions that would produce records/arrays/multisets (derived types) in their open format
// this is because constant folding them will make them closed (currently)
- if (function.getFunctionIdentifier().equals(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
+ if (fid.equals(BuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
if (returnType.getTypeTag() != ATypeTag.OBJECT || ((ARecordType) returnType).isOpen()
|| function.hasAnnotation(isTransformRecordAnnotation.class)) {
return false;
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index e354faf..fd3b50a 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -100,17 +100,12 @@
<hadoop.version>3.5.0</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.26.1</log4j.version>
-<<<<<<< HEAD (f7f02c [ASTERIXDB-3830][COMP] Enable hash-based COUNT(DISTINCT) by )
<avro.version>1.12.2</avro.version>
-=======
- <awsjavasdk.version>2.55.5</awsjavasdk.version>
- <s3mock.version>5.2.3</s3mock.version>
->>>>>>> BRANCH (6175d4 Merge branch 'gerrit/trinity' into 'gerrit/morpheus')
<parquet.version>1.18.1</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change -->
<hadoop-awsjavasdk.version>1.12.797</hadoop-awsjavasdk.version>
<!-- keep in sync with hyracks-fullstack/pom.xml -->
- <awsjavasdk.version>2.55.3</awsjavasdk.version>
+ <awsjavasdk.version>2.55.5</awsjavasdk.version>
<!-- @derived-from: software.amazon.awssdk:aws-sdk-java-pom:${awsjavasdk.version}#awscrt.version -->
<awsjavasdk.crt.version>0.48.4</awsjavasdk.crt.version>
@@ -121,12 +116,8 @@
<azurecore.version>1.59.1</azurecore.version>
<azurecorehttpnetty.version>1.16.7</azurecorehttpnetty.version>
-<<<<<<< HEAD (f7f02c [ASTERIXDB-3830][COMP] Enable hash-based COUNT(DISTINCT) by )
<!-- keep in sync with hyracks-fullstack/pom.xml -->
- <google-cloud-storage.version>2.73.0</google-cloud-storage.version>
-=======
- <gcsjavasdk.version>2.74.0</gcsjavasdk.version>
->>>>>>> BRANCH (6175d4 Merge branch 'gerrit/trinity' into 'gerrit/morpheus')
+ <google-cloud-storage.version>2.74.0</google-cloud-storage.version>
<hadoop-azuresdk.version>8.6.6</hadoop-azuresdk.version>
<hadoop-gcs.version>hadoop3-2.2.25</hadoop-gcs.version>
<io.opencensus.version>0.31.1</io.opencensus.version>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 75199d2..32e5e8b2 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -81,12 +81,12 @@
<netty.tcnative.version>2.0.84.Final</netty.tcnative.version>
<asm.version>9.10.1</asm.version>
<!-- keep in sync with asterixdb/pom.xml -->
- <awsjavasdk.version>2.54.17</awsjavasdk.version>
+ <awsjavasdk.version>2.55.5</awsjavasdk.version>
<!-- keep in sync with asterixdb/pom.xml -->
- <google-cloud-storage.version>2.73.0</google-cloud-storage.version>
+ <google-cloud-storage.version>2.74.0</google-cloud-storage.version>
<!-- @derived-from: com.google.cloud:google-cloud-storage:${google-cloud-storage.version}#dep:google-cloud-core -->
- <google-cloud-core.version>2.75.0</google-cloud-core.version>
+ <google-cloud-core.version>2.76.0</google-cloud-core.version>
<!-- keep in sync with asterixdb/pom.xml -->
<azurecore.version>1.59.1</azurecore.version>
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21905?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I740c30cb5c70937e675c7acac07ed5bde5d73812
Gerrit-Change-Number: 21905
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <mb...@apache.org>