You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/06/21 23:06:54 UTC
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1844
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
[CLUS][MTD] += ClusterStateManager pending removal
- storage format changes: no
- interface changes: no
Add ClusterStateManager API to enable marking nodes as pending removal, so
they are not considered when creating the node groups for new datasets
Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
3 files changed, 49 insertions(+), 11 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/1844/1
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index f7327b4..8a7d757 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -709,17 +709,19 @@
protected static String configureNodegroupForDataset(ICcApplicationContext appCtx, Map<String, String> hints,
String dataverseName, String datasetName, MetadataProvider metadataProvider) throws Exception {
- Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes();
+ Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes(true);
Set<String> selectedNodes = new LinkedHashSet<>();
String hintValue = hints.get(DatasetNodegroupCardinalityHint.NAME);
if (hintValue == null) {
selectedNodes.addAll(allNodes);
} else {
int nodegroupCardinality;
- boolean valid = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME,
- hints.get(DatasetNodegroupCardinalityHint.NAME)).first;
+ final Pair<Boolean, String> validation = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME,
+ hints.get(DatasetNodegroupCardinalityHint.NAME));
+ boolean valid = validation.first;
if (!valid) {
- throw new CompilationException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME);
+ throw new CompilationException("Incorrect use of hint '" + DatasetNodegroupCardinalityHint.NAME +
+ "': " + validation.second);
} else {
nodegroupCardinality = Integer.parseInt(hints.get(DatasetNodegroupCardinalityHint.NAME));
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
index 277587f..03bf704 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
@@ -22,6 +22,7 @@
import java.util.Set;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
import org.apache.hyracks.algebricks.common.utils.Pair;
/**
@@ -106,22 +107,20 @@
@Override
public Pair<Boolean, String> validateValue(ICcApplicationContext appCtx, String value) {
- boolean valid = true;
int intValue;
try {
intValue = Integer.parseInt(value);
if (intValue < 0) {
return new Pair<>(false, "Value must be >= 0");
}
- int numNodesInCluster = appCtx.getMetadataProperties().getNodeNames().size();
+ int numNodesInCluster = ClusterStateManager.INSTANCE.getParticipantNodes(true).size();
if (numNodesInCluster < intValue) {
return new Pair<>(false,
- "Value must be greater or equal to the existing number of nodes in cluster ("
+ "Value must less than or equal to the available number of nodes in cluster ("
+ numNodesInCluster + ")");
}
} catch (NumberFormatException nfe) {
- valid = false;
- return new Pair<>(valid, "Inappropriate value");
+ return new Pair<>(false, "Inappropriate value");
}
return new Pair<>(true, null);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 9c96d32..416e101 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -65,6 +65,7 @@
private static final Logger LOGGER = Logger.getLogger(ClusterStateManager.class.getName());
public static final ClusterStateManager INSTANCE = new ClusterStateManager();
private final Map<String, Map<IOption, Object>> activeNcConfiguration = new HashMap<>();
+ private Set<String> pendingRemoval = new HashSet<>();
private final Cluster cluster;
private ClusterState state = ClusterState.UNUSABLE;
@@ -73,8 +74,8 @@
private boolean globalRecoveryCompleted = false;
- private Map<String, ClusterPartition[]> node2PartitionsMap = null;
- private SortedMap<Integer, ClusterPartition> clusterPartitions = null;
+ private Map<String, ClusterPartition[]> node2PartitionsMap;
+ private SortedMap<Integer, ClusterPartition> clusterPartitions;
private String currentMetadataNode = null;
private boolean metadataNodeActive = false;
@@ -101,6 +102,7 @@
}
failedNodes.add(nodeId);
ftStrategy.notifyNodeFailure(nodeId);
+ pendingRemoval.remove(nodeId);
}
public synchronized void addNCConfiguration(String nodeId, Map<IOption, Object> configuration)
@@ -234,6 +236,14 @@
Set<String> participantNodes = new HashSet<>();
for (String pNode : activeNcConfiguration.keySet()) {
participantNodes.add(pNode);
+ }
+ return participantNodes;
+ }
+
+ public synchronized Set<String> getParticipantNodes(boolean excludePendingRemoval) {
+ Set<String> participantNodes = getParticipantNodes();
+ if (excludePendingRemoval) {
+ participantNodes.removeAll(pendingRemoval);
}
return participantNodes;
}
@@ -387,4 +397,31 @@
}
}
}
+
+ public synchronized void removePending(String nodeId) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Registering intention to remove node id " + nodeId);
+ }
+ if (!activeNcConfiguration.containsKey(nodeId)) {
+ LOGGER.warning("Cannot register unknown node " + nodeId + " for pending removal");
+ }
+ pendingRemoval.add(nodeId);
+ }
+
+ public boolean cancelRemovePending(String nodeId) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deregistering intention to remove node id " + nodeId);
+ }
+ if (!pendingRemoval.remove(nodeId)) {
+ LOGGER.warning("Cannot deregister intention to remove node id " + nodeId + " that was not registered");
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public synchronized Set<String> getNodesPendingRemoval() {
+ return new HashSet<>(pendingRemoval);
+ }
+
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/3265/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app-sql-execution/179/ (10/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
(4 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1844/1//COMMIT_MSG
Commit Message:
Line 7: [CLUS][MTD] += ClusterStateManager pending removal
> You might want to add a JIRA issue for this
Is there a new requirement to have JIRA issues for all changes?
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
File asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java:
PS1, Line 723: "
> Error code?
Not a regression.
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
File asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java:
PS1, Line 235: getParticipantNodes
> There are only 4 usages of this method. I think eventually all existing usa
>> I think eventually all existing usages will have to use the excludePendingRemoval flag
I don't understand this assertion. It seemed like other uses should reflect the state of the cluster...
PS1, Line 411: public boolean
> synchronized?
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2: Code-Review+2
sorry meant to +2 but was tired so saw the - as a +
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
(5 comments)
Sorry if it wasn't ready to be reviewed.
https://asterix-gerrit.ics.uci.edu/#/c/1844/1//COMMIT_MSG
Commit Message:
Line 7: [CLUS][MTD] += ClusterStateManager pending removal
You might want to add a JIRA issue for this
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
File asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java:
PS1, Line 723: "
Error code?
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
File asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java:
PS1, Line 119:
be
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
File asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java:
PS1, Line 235: getParticipantNodes
There are only 4 usages of this method. I think eventually all existing usages will have to use the excludePendingRemoval flag. Might as well just add the flag to this method and pass false in the current usages, then remove the other one.
PS1, Line 411: public boolean
synchronized?
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-no-installer-app/860/ (9/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
> (1 comment)
>
> These recent cluster state changes have been causing regressions.
To what changes and regressions are you referring?
> believe that is due in part to not having a fully thought and
> documented design in advanced
This is a simple change to avoid targeting new datasets to node groups that are about to leave the cluster. It's not rocket science.
>
> Such a critical part of the system must have a design document.
> Let's discuss before we proceed!
This has been discussed multiple times as part of rebalance.
This change has been +2d, please explain the -2?
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/3265/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
(2 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1844/1//COMMIT_MSG
Commit Message:
Line 7: [CLUS][MTD] += ClusterStateManager pending removal
> Is there a new requirement to have JIRA issues for all changes?
According to the latest proposal, yes. At least the way I understood it and what others have been doing since.
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
File asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java:
PS1, Line 235: getParticipantNodes
> >> I think eventually all existing usages will have to use the excludePendi
Which state of the cluster? with all nodes or excluding the ones pending removal?
My suggestion was based on the assumption that you didn't want to impact the existing usages and that's why I proposed passing false, but I think they all should exclude the ones pending removal except for special cases in which you need to do an operation on a node pending removal.
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-cancellation-test/175/ (7/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/1445/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-verify-storage/795/ (1/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/1445/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Yingyi Bu has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/1441/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app-sql-execution/185/ (10/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/1061/ (8/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/1441/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/3259/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-asterix-app/887/ (4/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/5730/ (7/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
(1 comment)
These recent cluster state changes have been causing regressions. I believe that is due in part to not having a fully thought and documented design in advanced.
Such a critical part of the system must have a design document. Let's discuss before we proceed!
https://asterix-gerrit.ics.uci.edu/#/c/1844/1/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
File asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java:
PS1, Line 723: "
> Not a regression.
Error message has been updated.
Please, use error code!
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-asterix-app/881/ (3/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/1054/ (8/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-sonar/4251/ (2/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/hyracks-gerrit/156/ (6/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-source-assemblies/445/ (4/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-verify-storage/789/ (1/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/5736/ (6/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Yingyi Bu has submitted this change and it was merged.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
[CLUS][MTD] += ClusterStateManager pending removal
- storage format changes: no
- interface changes: no
Add ClusterStateManager API to enable marking nodes as pending removal, so
they are not considered when creating the node groups for new datasets
Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1844
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
BAD: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
3 files changed, 49 insertions(+), 11 deletions(-)
Approvals:
abdullah alamoudi: Looks good to me, approved
Yingyi Bu: Looks good to me, approved
Jenkins: Verified; No violations found; No violations found; Verified
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index f7327b4..8a7d757 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -709,17 +709,19 @@
protected static String configureNodegroupForDataset(ICcApplicationContext appCtx, Map<String, String> hints,
String dataverseName, String datasetName, MetadataProvider metadataProvider) throws Exception {
- Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes();
+ Set<String> allNodes = ClusterStateManager.INSTANCE.getParticipantNodes(true);
Set<String> selectedNodes = new LinkedHashSet<>();
String hintValue = hints.get(DatasetNodegroupCardinalityHint.NAME);
if (hintValue == null) {
selectedNodes.addAll(allNodes);
} else {
int nodegroupCardinality;
- boolean valid = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME,
- hints.get(DatasetNodegroupCardinalityHint.NAME)).first;
+ final Pair<Boolean, String> validation = DatasetHints.validate(appCtx, DatasetNodegroupCardinalityHint.NAME,
+ hints.get(DatasetNodegroupCardinalityHint.NAME));
+ boolean valid = validation.first;
if (!valid) {
- throw new CompilationException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME);
+ throw new CompilationException("Incorrect use of hint '" + DatasetNodegroupCardinalityHint.NAME +
+ "': " + validation.second);
} else {
nodegroupCardinality = Integer.parseInt(hints.get(DatasetNodegroupCardinalityHint.NAME));
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
index 277587f..709f655 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
@@ -22,6 +22,7 @@
import java.util.Set;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
import org.apache.hyracks.algebricks.common.utils.Pair;
/**
@@ -106,22 +107,20 @@
@Override
public Pair<Boolean, String> validateValue(ICcApplicationContext appCtx, String value) {
- boolean valid = true;
int intValue;
try {
intValue = Integer.parseInt(value);
if (intValue < 0) {
return new Pair<>(false, "Value must be >= 0");
}
- int numNodesInCluster = appCtx.getMetadataProperties().getNodeNames().size();
+ int numNodesInCluster = ClusterStateManager.INSTANCE.getParticipantNodes(true).size();
if (numNodesInCluster < intValue) {
return new Pair<>(false,
- "Value must be greater or equal to the existing number of nodes in cluster ("
+ "Value must be less than or equal to the available number of nodes in cluster ("
+ numNodesInCluster + ")");
}
} catch (NumberFormatException nfe) {
- valid = false;
- return new Pair<>(valid, "Inappropriate value");
+ return new Pair<>(false, "Inappropriate value");
}
return new Pair<>(true, null);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 9c96d32..da341af 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -65,6 +65,7 @@
private static final Logger LOGGER = Logger.getLogger(ClusterStateManager.class.getName());
public static final ClusterStateManager INSTANCE = new ClusterStateManager();
private final Map<String, Map<IOption, Object>> activeNcConfiguration = new HashMap<>();
+ private Set<String> pendingRemoval = new HashSet<>();
private final Cluster cluster;
private ClusterState state = ClusterState.UNUSABLE;
@@ -73,8 +74,8 @@
private boolean globalRecoveryCompleted = false;
- private Map<String, ClusterPartition[]> node2PartitionsMap = null;
- private SortedMap<Integer, ClusterPartition> clusterPartitions = null;
+ private Map<String, ClusterPartition[]> node2PartitionsMap;
+ private SortedMap<Integer, ClusterPartition> clusterPartitions;
private String currentMetadataNode = null;
private boolean metadataNodeActive = false;
@@ -101,6 +102,7 @@
}
failedNodes.add(nodeId);
ftStrategy.notifyNodeFailure(nodeId);
+ pendingRemoval.remove(nodeId);
}
public synchronized void addNCConfiguration(String nodeId, Map<IOption, Object> configuration)
@@ -234,6 +236,14 @@
Set<String> participantNodes = new HashSet<>();
for (String pNode : activeNcConfiguration.keySet()) {
participantNodes.add(pNode);
+ }
+ return participantNodes;
+ }
+
+ public synchronized Set<String> getParticipantNodes(boolean excludePendingRemoval) {
+ Set<String> participantNodes = getParticipantNodes();
+ if (excludePendingRemoval) {
+ participantNodes.removeAll(pendingRemoval);
}
return participantNodes;
}
@@ -387,4 +397,31 @@
}
}
}
+
+ public synchronized void removePending(String nodeId) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Registering intention to remove node id " + nodeId);
+ }
+ if (!activeNcConfiguration.containsKey(nodeId)) {
+ LOGGER.warning("Cannot register unknown node " + nodeId + " for pending removal");
+ }
+ pendingRemoval.add(nodeId);
+ }
+
+ public synchronized boolean cancelRemovePending(String nodeId) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deregistering intention to remove node id " + nodeId);
+ }
+ if (!pendingRemoval.remove(nodeId)) {
+ LOGGER.warning("Cannot deregister intention to remove node id " + nodeId + " that was not registered");
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public synchronized Set<String> getNodesPendingRemoval() {
+ return new HashSet<>(pendingRemoval);
+ }
+
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-asterix-app/888/ (10/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2: Code-Review-2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1844
to look at the new patch set (#2).
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
[CLUS][MTD] += ClusterStateManager pending removal
- storage format changes: no
- interface changes: no
Add ClusterStateManager API to enable marking nodes as pending removal, so
they are not considered when creating the node groups for new datasets
Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java
M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
3 files changed, 49 insertions(+), 11 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/44/1844/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-cancellation-test/169/ (5/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-sonar/4257/ (3/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 1:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/3259/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/hyracks-gerrit/162/ (5/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-source-assemblies/451/ (2/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [CLUS][MTD] += ClusterStateManager pending removal
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [CLUS][MTD] += ClusterStateManager pending removal
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-no-installer-app/866/ (9/10)
--
To view, visit https://asterix-gerrit.ics.uci.edu/1844
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I509a7eed727be1e339c1b75b75f778fd222cc0e2
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Murtadha Hubail <hu...@gmail.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No