You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/06/26 19:13:45 UTC
(asterixdb) 04/07: [ASTERIXDB-3621][ASTERIXDB-3580][COMP] Use sameAs() for comparing nodes domain when getting partitions map
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9d8f9e53b31da8565f021c52921e1ae739e251e0
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Fri Jun 6 15:19:52 2025 -0700
[ASTERIXDB-3621][ASTERIXDB-3580][COMP] Use sameAs() for comparing nodes domain when getting partitions map
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
When getting the partitions map to be included in the required
partitioning property, use sameAs() nodes domain comparison
similar to PropertiesUtil.matchPartitioningProps() instead of
enforcing nodes order as well.
Ext-ref: MB-67128
Change-Id: Ie53228de01e7980cd96b39523d8c20da3d188860
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19903
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Tested-by: Hussain Towaileb <hu...@gmail.com>
---
.../app/function/TPCDSAllTablesDataGeneratorDatasource.java | 2 +-
.../app/function/TPCDSSingleTableDataGeneratorDatasource.java | 2 +-
.../org/apache/asterix/metadata/declared/FunctionDataSource.java | 5 ++---
.../org/apache/asterix/metadata/declared/MetadataProvider.java | 9 +--------
.../algebricks/core/algebra/metadata/IMetadataProvider.java | 5 +++++
.../core/algebra/properties/DefaultNodeGroupDomain.java | 7 -------
6 files changed, 10 insertions(+), 20 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index ac19b0057e..89d4120d49 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -70,7 +70,7 @@ public class TPCDSAllTablesDataGeneratorDatasource extends FunctionDataSource {
@Override
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- return md.getDataPartitioningProvider().getClusterLocations();
+ return md.getClusterLocations();
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 6551346ed5..46a9303e38 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -78,7 +78,7 @@ public class TPCDSSingleTableDataGeneratorDatasource extends FunctionDataSource
@Override
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- return md.getDataPartitioningProvider().getClusterLocations();
+ return md.getClusterLocations();
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 2322e53ac9..02ffc25be9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -127,9 +127,8 @@ public abstract class FunctionDataSource extends DataSource {
AlgebricksAbsolutePartitionConstraint locations);
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- String[] sortedLocations = md.getDataPartitioningProvider().getClusterLocations().getLocations();
- return new AlgebricksAbsolutePartitionConstraint(
- Arrays.stream(sortedLocations).distinct().toArray(String[]::new));
+ String[] locations = md.getClusterLocations().getLocations();
+ return new AlgebricksAbsolutePartitionConstraint(Arrays.stream(locations).distinct().toArray(String[]::new));
}
protected IDataParserFactory createDataParserFactory() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 71f479f5c3..941c8f7890 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -25,7 +25,6 @@ import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -1059,10 +1058,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
return dataPartitioningProvider.getClusterLocations();
}
- public DataPartitioningProvider getDataPartitioningProvider() {
- return dataPartitioningProvider;
- }
-
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDataLookupRuntime(
JobSpecification jobSpec, Dataset dataset, int[] ridIndexes, boolean retainInput,
IVariableTypeEnvironment typeEnv, IOperatorSchema opSchema, JobGenContext context,
@@ -1880,10 +1875,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
if (!(nodeDomain instanceof DefaultNodeGroupDomain inputDomain)) {
return null;
}
- String[] inputLocations = inputDomain.getSortedNodes();
AlgebricksAbsolutePartitionConstraint locations = dataPartitioningProvider.getClusterLocations();
- String[] clusterLocations = locations.getLocations();
- if (!Arrays.equals(inputLocations, clusterLocations)) {
+ if (!inputDomain.sameAs(new DefaultNodeGroupDomain(locations))) {
return null;
}
return dataPartitioningProvider.getPartitionsMap();
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 20fc9c54cf..a74f10057c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -201,6 +201,11 @@ public interface IMetadataProvider<S, I> {
boolean isBlockingOperatorDisabled();
+ /**
+ * Returns the cluster partitions map when the input node domain is the same as the cluster. Otherwise, null.
+ * @param nodeDomain input node domain
+ * @return the cluster partitions map.
+ */
int[][] getPartitionsMap(INodeDomain nodeDomain);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
index 5921eea161..f785fec24d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.algebricks.core.algebra.properties;
-import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.MultiSet;
@@ -73,10 +72,4 @@ public class DefaultNodeGroupDomain implements INodeDomain {
public String[] getNodes() {
return nodes.toArray(new String[0]);
}
-
- public String[] getSortedNodes() {
- String[] sortedNodes = nodes.toArray(new String[0]);
- Arrays.sort(sortedNodes);
- return sortedNodes;
- }
}