You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/05/05 23:10:27 UTC
Change in asterixdb[cheshire-cat]: [NO ISSUE][OTH] Only order nodes in node group for new datasets
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343 )
Change subject: [NO ISSUE][OTH] Only order nodes in node group for new datasets
......................................................................
[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
6 files changed, 30 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/11343/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Gerrit-Change-Number: 11343
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: [NO ISSUE][OTH] Only order nodes in node group for new datasets
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Michael Blow <mb...@apache.org>:
Michael Blow has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343 )
Change subject: [NO ISSUE][OTH] Only order nodes in node group for new datasets
......................................................................
Patch Set 1: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Gerrit-Change-Number: 11343
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Comment-Date: Thu, 06 May 2021 13:45:51 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment
Change in asterixdb[cheshire-cat]: [NO ISSUE][OTH] Only order nodes in node group for new datasets
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343 )
Change subject: [NO ISSUE][OTH] Only order nodes in node group for new datasets
......................................................................
[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
6 files changed, 30 insertions(+), 16 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/11343/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Gerrit-Change-Number: 11343
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-MessageType: newchange
Change in asterixdb[cheshire-cat]: [NO ISSUE][OTH] Only order nodes in node group for new datasets
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343 )
Change subject: [NO ISSUE][OTH] Only order nodes in node group for new datasets
......................................................................
[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
6 files changed, 30 insertions(+), 16 deletions(-)
Approvals:
Michael Blow: Looks good to me, approved
Jenkins: Verified; Verified
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Gerrit-Change-Number: 11343
Gerrit-PatchSet: 2
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-MessageType: merged
Change in asterixdb[cheshire-cat]: [NO ISSUE][OTH] Only order nodes in node group for new datasets
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Jenkins <je...@fulliautomatix.ics.uci.edu>:
Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343 )
Change subject: [NO ISSUE][OTH] Only order nodes in node group for new datasets
......................................................................
Patch Set 1: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/11949/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Gerrit-Change-Number: 11343
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Thu, 06 May 2021 00:11:51 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment