You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2022/02/02 18:55:32 UTC
[asterixdb] 03/12: [NO ISSUE][CONF] Removed unused active partitions config
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5f52a5f7059691be915d781e9370d5295273dded
Author: Murtadha Hubail <mu...@couchbase.com>
AuthorDate: Thu Jan 27 23:17:03 2022 +0300
[NO ISSUE][CONF] Removed unused active partitions config
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Remove unused active partitions config.
- Improved node partitions logging.
- Logging fixes.
Change-Id: I4f2a611e2846405738401310f485db0e72844031
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/15024
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 2 +-
.../org/apache/asterix/app/nc/ReplicaManager.java | 11 +++--------
.../asterix/app/nc/task/UpdateNodeStatusTask.java | 4 ++--
.../app/replication/NcLifecycleCoordinator.java | 5 +++--
.../message/NCLifecycleTaskReportMessage.java | 6 ++++++
.../message/RegistrationTasksRequestMessage.java | 20 +++++++++++++-------
.../message/RegistrationTasksResponseMessage.java | 8 ++------
.../asterix/hyracks/bootstrap/NCApplication.java | 4 ++--
.../asterix/common/config/MetadataProperties.java | 4 ++--
.../apache/asterix/common/config/NodeProperties.java | 3 +--
.../asterix/common/config/PropertiesAccessor.java | 18 +++++++-----------
.../apache/asterix/common/utils/NcLocalCounters.java | 6 ++++++
12 files changed, 48 insertions(+), 43 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index f532352..1a89168 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -224,7 +224,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
new DatasetLifecycleManager(storageProperties, localResourceRepository, txnSubsystem.getLogManager(),
virtualBufferCache, indexCheckpointManagerProvider, ioManager.getIODevices().size());
final String nodeId = getServiceContext().getNodeId();
- final Set<Integer> nodePartitions = metadataProperties.getNodeActivePartitions(nodeId);
+ final Set<Integer> nodePartitions = metadataProperties.getNodePartitions(nodeId);
replicaManager = new ReplicaManager(this, nodePartitions);
isShuttingdown = false;
activeManager = new ActiveManager(threadExecutor, getServiceContext().getNodeId(),
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 7c4b59c..7b52df6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -19,7 +19,6 @@
package org.apache.asterix.app.nc;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -31,7 +30,6 @@ import java.util.stream.Stream;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.replication.IPartitionReplica;
import org.apache.asterix.common.storage.IReplicaManager;
import org.apache.asterix.common.storage.ReplicaIdentifier;
@@ -188,11 +186,8 @@ public class ReplicaManager implements IReplicaManager {
}
private void setNodeOwnedPartitions(INcApplicationContext appCtx) {
- ClusterPartition[] clusterPartitions =
- appCtx.getMetadataProperties().getNodePartitions().get(appCtx.getServiceContext().getNodeId());
- if (clusterPartitions != null) {
- nodeOwnedPartitions.addAll(Arrays.stream(clusterPartitions).map(ClusterPartition::getPartitionId)
- .collect(Collectors.toList()));
- }
+ Set<Integer> nodePartitions =
+ appCtx.getMetadataProperties().getNodePartitions(appCtx.getServiceContext().getNodeId());
+ nodeOwnedPartitions.addAll(nodePartitions);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
index fe579ad..17eff4a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
@@ -36,7 +36,7 @@ public class UpdateNodeStatusTask implements INCLifecycleTask {
private static final Logger LOGGER = LogManager.getLogger();
private static final long serialVersionUID = 2L;
private final NodeStatus status;
- private Set<Integer> activePartitions;
+ private final Set<Integer> activePartitions;
public UpdateNodeStatusTask(NodeStatus status, Set<Integer> activePartitions) {
this.status = status;
@@ -61,6 +61,6 @@ public class UpdateNodeStatusTask implements INCLifecycleTask {
@Override
public String toString() {
- return "{ \"class\" : \"" + getClass().getSimpleName() + "\" }";
+ return "UpdateNodeStatusTask{" + "status=" + status + ", activePartitions=" + activePartitions + '}';
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 22a0a84..06005a9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -168,8 +168,9 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
protected List<INCLifecycleTask> buildNCRegTasks(String nodeId, NodeStatus nodeStatus, SystemState state,
Set<Integer> activePartitions) {
- LOGGER.info("Building registration tasks for node {} with status {} and system state: {}", nodeId, nodeStatus,
- state);
+ LOGGER.info(
+ "Building registration tasks for node {} with status {} and system state: {} and active partitions {}",
+ nodeId, nodeStatus, state, activePartitions);
final boolean isMetadataNode = nodeId.equals(metadataNodeId);
switch (nodeStatus) {
case ACTIVE:
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
index 1309369..b2a2dd7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
@@ -76,4 +76,10 @@ public class NCLifecycleTaskReportMessage implements INCLifecycleMessage, ICcAdd
public Set<Integer> getActivePartitions() {
return activePartitions;
}
+
+ @Override
+ public String toString() {
+ return "NCLifecycleTaskReportMessage{" + "nodeId='" + nodeId + '\'' + ", success=" + success + ", exception="
+ + exception + ", localCounters=" + localCounters + ", activePartitions=" + activePartitions + '}';
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
index fb50b3e..9e95ac6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.app.replication.message;
-import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -39,18 +38,18 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc
private static final Logger LOGGER = LogManager.getLogger();
private static final long serialVersionUID = 2L;
- protected final SystemState state;
- protected final String nodeId;
- protected final NodeStatus nodeStatus;
- protected final Map<String, Object> secrets;
- protected final Set<Integer> activePartitions;
+ private final SystemState state;
+ private final String nodeId;
+ private final NodeStatus nodeStatus;
+ private final Map<String, Object> secrets;
+ private final Set<Integer> activePartitions;
public RegistrationTasksRequestMessage(String nodeId, NodeStatus nodeStatus, SystemState state,
Map<String, Object> secretsEphemeral, Set<Integer> activePartitions) {
this.state = state;
this.nodeId = nodeId;
this.nodeStatus = nodeStatus;
- this.secrets = new HashMap<>(secretsEphemeral);
+ this.secrets = secretsEphemeral;
this.activePartitions = activePartitions;
}
@@ -59,6 +58,7 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc
try {
RegistrationTasksRequestMessage msg = new RegistrationTasksRequestMessage(cs.getId(), nodeStatus,
systemState, secretsEphemeral, activePartitions);
+ LOGGER.info("sending {} to CC", msg);
((INCMessageBroker) cs.getContext().getMessageBroker()).sendMessageToCC(ccId, msg);
} catch (Exception e) {
LOGGER.log(Level.ERROR, "Unable to send RegistrationTasksRequestMessage to CC", e);
@@ -95,4 +95,10 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc
public Set<Integer> getActivePartitions() {
return activePartitions;
}
+
+ @Override
+ public String toString() {
+ return "RegistrationTasksRequestMessage{" + "state=" + state + ", nodeId='" + nodeId + '\'' + ", nodeStatus="
+ + nodeStatus + ", activePartitions=" + activePartitions + '}';
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
index f0a4a7c..f0f0470 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
@@ -59,13 +59,9 @@ public class RegistrationTasksResponseMessage extends CcIdentifiedMessage
Throwable exception = null;
try {
for (INCLifecycleTask task : tasks) {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.log(Level.INFO, "Starting startup task: " + task);
- }
+ LOGGER.log(Level.INFO, "Starting startup task: {}", task);
task.perform(getCcId(), cs);
- if (LOGGER.isInfoEnabled()) {
- LOGGER.log(Level.INFO, "Completed startup task: " + task);
- }
+ LOGGER.log(Level.INFO, "Completed startup task: {}", task);
}
} catch (Throwable e) { //NOSONAR all startup failures should be reported to CC
LOGGER.log(Level.ERROR, "Failed during startup task", e);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index be1cc7c..f60349f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -300,12 +300,12 @@ public class NCApplication extends BaseNCApplication {
final NodeStatus currentStatus = ncs.getNodeStatus();
final SystemState systemState = isPendingStartupTasks(currentStatus, ncs.getPrimaryCcId(), ccId)
? getCurrentSystemState() : SystemState.HEALTHY;
- final Map httpSecrets =
+ final Map<String, Object> httpSecrets =
apiServer != null ? Collections.singletonMap(SYS_AUTH_HEADER, apiServer.ctx().get(SYS_AUTH_HEADER))
: Collections.emptyMap();
RegistrationTasksRequestMessage.send(ccId, (NodeControllerService) ncServiceCtx.getControllerService(),
currentStatus, systemState, httpSecrets,
- runtimeContext.getMetadataProperties().getNodeActivePartitions(nodeId));
+ runtimeContext.getMetadataProperties().getNodePartitions(nodeId));
}
@Override
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
index 31708d3..252017f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
@@ -124,8 +124,8 @@ public class MetadataProperties extends AbstractProperties {
return accessor.getClusterPartitions();
}
- public Set<Integer> getNodeActivePartitions(String nodeId) {
- return accessor.getActivePartitions(nodeId);
+ public Set<Integer> getNodePartitions(String nodeId) {
+ return accessor.getNodePartitions(nodeId);
}
public Map<String, String> getTransactionLogDirs() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
index afb103d..522cabd 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
@@ -46,8 +46,7 @@ public class NodeProperties extends AbstractProperties {
STARTING_PARTITION_ID(
OptionTypes.INTEGER,
-1,
- "The first partition id to assign to iodevices on this node (-1 == auto-assign)"),
- ACTIVE_PARTITIONS(OptionTypes.STRING_ARRAY, null, "List of node active partitions");
+ "The first partition id to assign to iodevices on this node (-1 == auto-assign)");
private final IOptionType type;
private final Object defaultValue;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
index 5ba378d..80f9a17 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
@@ -41,6 +41,7 @@ import java.util.stream.Collectors;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.utils.PrintUtil;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.config.IApplicationConfig;
@@ -69,7 +70,7 @@ public class PropertiesAccessor implements IApplicationConfig {
/**
* Constructor which wraps an IApplicationConfig.
*/
- private PropertiesAccessor(IApplicationConfig cfg) throws AsterixException, IOException {
+ private PropertiesAccessor(IApplicationConfig cfg) throws AsterixException {
this.cfg = cfg;
nodePartitionsMap = new ConcurrentHashMap<>();
clusterPartitions = Collections.synchronizedSortedMap(new TreeMap<>());
@@ -80,6 +81,7 @@ public class PropertiesAccessor implements IApplicationConfig {
for (String ncName : cfg.getNCNames()) {
configureNc(configManager, ncName, uniquePartitionId);
}
+ LOGGER.info("configured partitions: {} from config {}", () -> PrintUtil.toString(nodePartitionsMap), () -> cfg);
for (String section : cfg.getSectionNames()) {
if (section.startsWith(AsterixProperties.SECTION_PREFIX_EXTENSION)) {
String className = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_EXTENSION, section);
@@ -194,22 +196,16 @@ public class PropertiesAccessor implements IApplicationConfig {
return clusterPartitions;
}
- public Set<Integer> getActivePartitions(String nodeId) {
- // by default, node actives partitions are the partitions assigned to the node
- String[] activePartitions = cfg.getStringArray(NodeProperties.Option.ACTIVE_PARTITIONS);
- if (activePartitions == null) {
- ClusterPartition[] nodeClusterPartitions = nodePartitionsMap.get(nodeId);
- return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId)
- .collect(Collectors.toSet());
- }
- return Arrays.stream(activePartitions).map(Integer::parseInt).collect(Collectors.toSet());
+ public Set<Integer> getNodePartitions(String nodeId) {
+ ClusterPartition[] nodeClusterPartitions = nodePartitionsMap.get(nodeId);
+ return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId).collect(Collectors.toSet());
}
public List<AsterixExtension> getExtensions() {
return extensions;
}
- public static PropertiesAccessor getInstance(IApplicationConfig cfg) throws IOException, AsterixException {
+ public static PropertiesAccessor getInstance(IApplicationConfig cfg) throws AsterixException {
PropertiesAccessor accessor = instances.get(cfg);
if (accessor == null) {
accessor = new PropertiesAccessor(cfg);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java
index 5cf6724..8956b93 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java
@@ -59,4 +59,10 @@ public class NcLocalCounters implements Serializable {
public long getMaxJobId() {
return maxJobId;
}
+
+ @Override
+ public String toString() {
+ return "NcLocalCounters{" + "maxResourceId=" + maxResourceId + ", maxTxnId=" + maxTxnId + ", maxJobId="
+ + maxJobId + '}';
+ }
}