You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by je...@apache.org on 2023/12/03 13:11:46 UTC
(incubator-hugegraph) 01/02: enhence ServerInfoManager.heartbeat()
This is an automated email from the ASF dual-hosted git repository.
jermy pushed a commit to branch server-info-to-global
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
commit f4c0f4c160be0fc25c100609b2c034ae0e76d7c2
Author: Jermy Li <je...@apache.org>
AuthorDate: Sun Dec 3 17:01:19 2023 +0800
enhence ServerInfoManager.heartbeat()
Change-Id: I6932893c4be8331547f3b721083ca00430f85e58
---
.../apache/hugegraph/auth/HugeGraphAuthProxy.java | 8 +-
.../org/apache/hugegraph/core/GraphManager.java | 14 +--
.../main/java/org/apache/hugegraph/HugeGraph.java | 4 +-
.../org/apache/hugegraph/StandardHugeGraph.java | 14 +--
.../hugegraph/masterelection/GlobalMasterInfo.java | 42 ++++----
.../apache/hugegraph/task/ServerInfoManager.java | 113 +++++++++++++--------
.../hugegraph/task/StandardTaskScheduler.java | 14 +--
.../org/apache/hugegraph/task/TaskManager.java | 8 +-
8 files changed, 121 insertions(+), 96 deletions(-)
diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java
index 6fa34ff18..d22b86bab 100644
--- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java
+++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java
@@ -669,9 +669,9 @@ public final class HugeGraphAuthProxy implements HugeGraph {
}
@Override
- public void serverStarted(GlobalMasterInfo serverInfo) {
+ public void serverStarted(GlobalMasterInfo nodeInfo) {
this.verifyAdminPermission();
- this.hugegraph.serverStarted(serverInfo);
+ this.hugegraph.serverStarted(nodeInfo);
}
@Override
@@ -776,9 +776,9 @@ public final class HugeGraphAuthProxy implements HugeGraph {
}
@Override
- public void create(String configPath, GlobalMasterInfo serverInfo) {
+ public void create(String configPath, GlobalMasterInfo nodeInfo) {
this.verifyPermission(HugePermission.WRITE, ResourceType.STATUS);
- this.hugegraph.create(configPath, serverInfo);
+ this.hugegraph.create(configPath, nodeInfo);
}
@Override
diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
index 5ca7d0a0f..b8770ca7d 100644
--- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
+++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java
@@ -455,23 +455,23 @@ public final class GraphManager {
}
private void serverStarted(HugeConfig config) {
- String serverId = config.get(ServerOptions.SERVER_ID);
+ String id = config.get(ServerOptions.SERVER_ID);
String role = config.get(ServerOptions.SERVER_ROLE);
- E.checkArgument(StringUtils.isNotEmpty(serverId),
+ E.checkArgument(StringUtils.isNotEmpty(id),
"The server name can't be null or empty");
E.checkArgument(StringUtils.isNotEmpty(role),
"The server role can't be null or empty");
- NodeRole serverRole = NodeRole.valueOf(role.toUpperCase());
- boolean supportRoleElection = !serverRole.computer() &&
+ NodeRole nodeRole = NodeRole.valueOf(role.toUpperCase());
+ boolean supportRoleElection = !nodeRole.computer() &&
this.supportRoleElection();
if (supportRoleElection) {
// Init any server as Worker role, then do role election
- serverRole = NodeRole.WORKER;
+ nodeRole = NodeRole.WORKER;
}
- this.globalNodeRoleInfo.serverId(IdGenerator.of(serverId));
- this.globalNodeRoleInfo.initServerRole(serverRole);
+ this.globalNodeRoleInfo.initNodeId(IdGenerator.of(id));
+ this.globalNodeRoleInfo.initNodeRole(nodeRole);
for (String graph : this.graphs()) {
HugeGraph hugegraph = this.graph(graph);
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java
index 420f8bd4d..85c093e59 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeGraph.java
@@ -201,7 +201,7 @@ public interface HugeGraph extends Graph {
void waitReady(RpcServer rpcServer);
- void serverStarted(GlobalMasterInfo serverInfo);
+ void serverStarted(GlobalMasterInfo nodeInfo);
boolean started();
@@ -221,7 +221,7 @@ public interface HugeGraph extends Graph {
void resumeSnapshot();
- void create(String configPath, GlobalMasterInfo serverInfo);
+ void create(String configPath, GlobalMasterInfo nodeInfo);
void drop();
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java
index df3de8671..c671056e0 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java
@@ -267,15 +267,15 @@ public class StandardHugeGraph implements HugeGraph {
}
@Override
- public void serverStarted(GlobalMasterInfo serverInfo) {
+ public void serverStarted(GlobalMasterInfo nodeInfo) {
LOG.info("Init system info for graph '{}'", this.name);
this.initSystemInfo();
LOG.info("Init server info [{}-{}] for graph '{}'...",
- serverInfo.serverId(), serverInfo.serverRole(), this.name);
- this.serverInfoManager().initServerInfo(serverInfo);
+ nodeInfo.nodeId(), nodeInfo.nodeRole(), this.name);
+ this.serverInfoManager().initServerInfo(nodeInfo);
- this.initRoleStateMachine(serverInfo.serverId());
+ this.initRoleStateMachine(nodeInfo.nodeId());
// TODO: check necessary?
LOG.info("Check olap property-key tables for graph '{}'", this.name);
@@ -401,7 +401,7 @@ public class StandardHugeGraph implements HugeGraph {
try {
this.storeProvider.truncate();
// TODO: remove this after serverinfo saved in etcd
- this.serverStarted(this.serverInfoManager().serverInfo());
+ this.serverStarted(this.serverInfoManager().globalNodeRoleInfo());
} finally {
LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK);
}
@@ -975,9 +975,9 @@ public class StandardHugeGraph implements HugeGraph {
}
@Override
- public void create(String configPath, GlobalMasterInfo serverInfo) {
+ public void create(String configPath, GlobalMasterInfo nodeInfo) {
this.initBackend();
- this.serverStarted(serverInfo);
+ this.serverStarted(nodeInfo);
// Write config to disk file
String confPath = ConfigUtil.writeToFile(configPath, this.name(),
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java
index 709b1b75d..9124fcd0b 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/masterelection/GlobalMasterInfo.java
@@ -30,8 +30,8 @@ public final class GlobalMasterInfo {
private volatile boolean supportElection;
private volatile NodeInfo masterNodeInfo;
- private volatile Id serverId;
- private volatile NodeRole serverRole;
+ private volatile Id nodeId;
+ private volatile NodeRole nodeRole;
public GlobalMasterInfo() {
this(NO_MASTER);
@@ -41,8 +41,8 @@ public final class GlobalMasterInfo {
this.supportElection = false;
this.masterNodeInfo = masterInfo;
- this.serverId = null;
- this.serverRole = null;
+ this.nodeId = null;
+ this.nodeRole = null;
}
public void supportElection(boolean featureSupport) {
@@ -66,36 +66,36 @@ public final class GlobalMasterInfo {
return this.masterNodeInfo;
}
- public Id serverId() {
- return this.serverId;
+ public Id nodeId() {
+ return this.nodeId;
}
- public NodeRole serverRole() {
- return this.serverRole;
+ public NodeRole nodeRole() {
+ return this.nodeRole;
}
- public void serverId(Id id) {
- this.serverId = id;
+ public void initNodeId(Id id) {
+ this.nodeId = id;
}
- public void initServerRole(NodeRole role) {
+ public void initNodeRole(NodeRole role) {
E.checkArgument(role != null, "The server role can't be null");
- E.checkArgument(this.serverRole == null,
+ E.checkArgument(this.nodeRole == null,
"The server role can't be init twice");
- this.serverRole = role;
+ this.nodeRole = role;
}
- public void changeServerRole(NodeRole role) {
+ public void changeNodeRole(NodeRole role) {
E.checkArgument(role != null, "The server role can't be null");
- this.serverRole = role;
+ this.nodeRole = role;
}
- public static GlobalMasterInfo master(String serverId) {
- NodeInfo masterInfo = new NodeInfo(true, serverId);
- GlobalMasterInfo serverInfo = new GlobalMasterInfo(masterInfo);
- serverInfo.serverId = IdGenerator.of(serverId);
- serverInfo.serverRole = NodeRole.MASTER;
- return serverInfo;
+ public static GlobalMasterInfo master(String nodeId) {
+ NodeInfo masterInfo = new NodeInfo(true, nodeId);
+ GlobalMasterInfo nodeInfo = new GlobalMasterInfo(masterInfo);
+ nodeInfo.nodeId = IdGenerator.of(nodeId);
+ nodeInfo.nodeRole = NodeRole.MASTER;
+ return nodeInfo;
}
public static class NodeInfo {
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java
index edf3e4973..ee14b4ceb 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/ServerInfoManager.java
@@ -62,7 +62,7 @@ public class ServerInfoManager {
private final HugeGraphParams graph;
private final ExecutorService dbExecutor;
- private GlobalMasterInfo globalServerInfo;
+ private volatile GlobalMasterInfo globalNodeInfo;
private volatile boolean onlySingleNode;
private volatile boolean closed;
@@ -75,7 +75,7 @@ public class ServerInfoManager {
this.graph = graph;
this.dbExecutor = dbExecutor;
- this.globalServerInfo = null;
+ this.globalNodeInfo = null;
this.onlySingleNode = false;
this.closed = false;
@@ -85,7 +85,7 @@ public class ServerInfoManager {
HugeServerInfo.schema(this.graph).initSchemaIfNeeded();
}
- public boolean close() {
+ public synchronized boolean close() {
this.closed = true;
if (!this.dbExecutor.isShutdown()) {
this.removeSelfServerInfo();
@@ -102,16 +102,16 @@ public class ServerInfoManager {
return true;
}
- public synchronized void initServerInfo(GlobalMasterInfo serverInfo) {
- E.checkArgument(serverInfo != null, "The global node info can't be null");
- this.globalServerInfo = serverInfo;
- Id serverId = this.globalServerInfo.serverId();
+ public synchronized void initServerInfo(GlobalMasterInfo nodeInfo) {
+ E.checkArgument(nodeInfo != null, "The global node info can't be null");
+ Id serverId = nodeInfo.nodeId();
HugeServerInfo existed = this.serverInfo(serverId);
E.checkArgument(existed == null || !existed.alive(),
"The server with name '%s' already in cluster",
serverId);
- if (this.globalServerInfo.serverRole().master()) {
+
+ if (nodeInfo.nodeRole().master()) {
String page = this.supportsPaging() ? PageInfo.PAGE_NONE : null;
do {
Iterator<HugeServerInfo> servers = this.serverInfos(PAGE_SIZE, page);
@@ -127,55 +127,80 @@ public class ServerInfoManager {
} while (page != null);
}
- // TODO: save ServerInfo at AuthServer
- this.saveServerInfo(this.selfServerId(), this.selfServerRole());
+ this.globalNodeInfo = nodeInfo;
+
+ // TODO: save ServerInfo to AuthServer
+ this.saveServerInfo(this.selfNodeId(), this.selfNodeRole());
}
- public synchronized void changeServerRole(NodeRole serverRole) {
+ public synchronized void changeServerRole(NodeRole nodeRole) {
if (this.closed) {
return;
}
- this.globalServerInfo.changeServerRole(serverRole);
+ this.globalNodeInfo.changeNodeRole(nodeRole);
- this.saveServerInfo(this.selfServerId(), this.selfServerRole());
+ // TODO: save ServerInfo to AuthServer
+ this.saveServerInfo(this.selfNodeId(), this.selfNodeRole());
}
- public GlobalMasterInfo serverInfo() {
- return this.globalServerInfo;
+ public GlobalMasterInfo globalNodeRoleInfo() {
+ return this.globalNodeInfo;
}
- public Id selfServerId() {
- if (this.globalServerInfo == null) {
+ public Id selfNodeId() {
+ if (this.globalNodeInfo == null) {
return null;
}
- return this.globalServerInfo.serverId();
+ return this.globalNodeInfo.nodeId();
}
- public NodeRole selfServerRole() {
- if (this.globalServerInfo == null) {
+ public NodeRole selfNodeRole() {
+ if (this.globalNodeInfo == null) {
return null;
}
- return this.globalServerInfo.serverRole();
+ return this.globalNodeInfo.nodeRole();
}
- public boolean master() {
- return this.selfServerRole() != null && this.selfServerRole().master();
+ public boolean selfIsMaster() {
+ return this.selfNodeRole() != null && this.selfNodeRole().master();
}
public boolean onlySingleNode() {
- // Only has one master node
+ // Only exists one node in the whole master
return this.onlySingleNode;
}
- public void heartbeat() {
+ public synchronized void heartbeat() {
+ assert this.graphIsReady();
+
HugeServerInfo serverInfo = this.selfServerInfo();
- if (serverInfo == null && this.selfServerId() != null &&
- this.selfServerRole() != NodeRole.MASTER) {
- serverInfo = this.saveServerInfo(this.selfServerId(), this.selfServerRole());
+ if (serverInfo != null) {
+ // Update heartbeat time for this server
+ serverInfo.updateTime(DateUtil.now());
+ this.save(serverInfo);
+ return;
}
- serverInfo.updateTime(DateUtil.now());
- this.save(serverInfo);
+
+ /* ServerInfo is missing */
+ if (this.selfNodeId() == null) {
+ // Ignore if ServerInfo is not initialized
+ LOG.info("ServerInfo is missing: {}, may not be initialized yet");
+ return;
+ }
+ if (this.selfIsMaster()) {
+ // On master node, just wait for ServerInfo re-init
+ LOG.warn("ServerInfo is missing: {}, may be cleared before",
+ this.selfNodeId());
+ return;
+ }
+ /*
+ * Missing server info on non-master node, may be caused by graph
+ * truncated on master node then synced by raft.
+ * TODO: we just patch it here currently, to be improved.
+ */
+ serverInfo = this.saveServerInfo(this.selfNodeId(), this.selfNodeRole());
+ assert serverInfo != null;
}
public synchronized void decreaseLoad(int load) {
@@ -190,7 +215,7 @@ public class ServerInfoManager {
return 10000;
}
- protected boolean graphReady() {
+ protected boolean graphIsReady() {
return !this.closed && this.graph.started() && this.graph.initialized();
}
@@ -244,8 +269,8 @@ public class ServerInfoManager {
return this.graph.systemTransaction();
}
- private HugeServerInfo saveServerInfo(Id server, NodeRole role) {
- HugeServerInfo serverInfo = new HugeServerInfo(server, role);
+ private HugeServerInfo saveServerInfo(Id serverId, NodeRole serverRole) {
+ HugeServerInfo serverInfo = new HugeServerInfo(serverId, serverRole);
serverInfo.maxLoad(this.calcMaxLoad());
this.save(serverInfo);
@@ -312,16 +337,16 @@ public class ServerInfoManager {
}
private HugeServerInfo selfServerInfo() {
- HugeServerInfo selfServerInfo = this.serverInfo(this.selfServerId());
- if (selfServerInfo == null) {
- LOG.warn("ServerInfo is missing: {}", this.selfServerId());
+ HugeServerInfo selfServerInfo = this.serverInfo(this.selfNodeId());
+ if (selfServerInfo == null && this.selfNodeId() != null) {
+ LOG.warn("ServerInfo is missing: {}", this.selfNodeId());
}
return selfServerInfo;
}
- private HugeServerInfo serverInfo(Id server) {
+ private HugeServerInfo serverInfo(Id serverId) {
return this.call(() -> {
- Iterator<Vertex> vertices = this.tx().queryVertices(server);
+ Iterator<Vertex> vertices = this.tx().queryVertices(serverId);
Vertex vertex = QueryResults.one(vertices);
if (vertex == null) {
return null;
@@ -337,19 +362,19 @@ public class ServerInfoManager {
* backend store, initServerInfo() is not called in this case, so
* this.selfServerId is null at this time.
*/
- if (this.selfServerId() != null && this.graph.initialized()) {
- return this.removeServerInfo(this.selfServerId());
+ if (this.selfNodeId() != null && this.graph.initialized()) {
+ return this.removeServerInfo(this.selfNodeId());
}
return null;
}
- private HugeServerInfo removeServerInfo(Id server) {
- if (server == null) {
+ private HugeServerInfo removeServerInfo(Id serverId) {
+ if (serverId == null) {
return null;
}
- LOG.info("Remove server info: {}", server);
+ LOG.info("Remove server info: {}", serverId);
return this.call(() -> {
- Iterator<Vertex> vertices = this.tx().queryVertices(server);
+ Iterator<Vertex> vertices = this.tx().queryVertices(serverId);
Vertex vertex = QueryResults.one(vertices);
if (vertex == null) {
return null;
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java
index bf0a16314..120aeb0d6 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java
@@ -141,7 +141,7 @@ public class StandardTaskScheduler implements TaskScheduler {
@Override
public <V> void restoreTasks() {
- Id selfServer = this.serverManager().selfServerId();
+ Id selfServer = this.serverManager().selfNodeId();
// Restore 'RESTORING', 'RUNNING' and 'QUEUED' tasks in order.
for (TaskStatus status : TaskStatus.PENDING_STATUSES) {
String page = this.supportsPaging() ? PageInfo.PAGE_NONE : null;
@@ -205,7 +205,7 @@ public class StandardTaskScheduler implements TaskScheduler {
* this code can be removed without affecting code logic
*/
task.status(TaskStatus.QUEUED);
- task.server(this.serverManager().selfServerId());
+ task.server(this.serverManager().selfNodeId());
this.save(task);
return this.submitTask(task);
} else {
@@ -278,8 +278,8 @@ public class StandardTaskScheduler implements TaskScheduler {
// The task scheduled to workers, let the worker node to cancel
this.save(task);
assert task.server() != null : task;
- assert this.serverManager().master();
- if (!task.server().equals(this.serverManager().selfServerId())) {
+ assert this.serverManager().selfIsMaster();
+ if (!task.server().equals(this.serverManager().selfNodeId())) {
/*
* Remove task from memory if it's running on worker node,
* but keep task in memory if it's running on master node.
@@ -318,7 +318,7 @@ public class StandardTaskScheduler implements TaskScheduler {
continue;
}
- if (!this.serverManager.master()) {
+ if (!this.serverManager.selfIsMaster()) {
return;
}
@@ -423,7 +423,7 @@ public class StandardTaskScheduler implements TaskScheduler {
protected void taskDone(HugeTask<?> task) {
this.remove(task);
- Id selfServerId = this.serverManager().selfServerId();
+ Id selfServerId = this.serverManager().selfNodeId();
try {
this.serverManager().decreaseLoad(task.load());
} catch (Throwable e) {
@@ -719,7 +719,7 @@ public class StandardTaskScheduler implements TaskScheduler {
}
private void checkOnMasterNode(String op) {
- if (!this.serverManager().master()) {
+ if (!this.serverManager().selfIsMaster()) {
throw new HugeException("Can't %s task on non-master server", op);
}
}
diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java
index bb3af44f4..b3726d383 100644
--- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java
+++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/TaskManager.java
@@ -324,7 +324,7 @@ public final class TaskManager {
* If graph is closed, don't call serverManager.initialized()
* due to it will reopen graph tx.
*/
- if (!serverManager.graphReady()) {
+ if (!serverManager.graphIsReady()) {
return;
}
@@ -338,7 +338,7 @@ public final class TaskManager {
* However, when enableRoleElected=false, a Master is only set by the
* config assignment, assigned-Master always stays the same state.
*/
- if (serverManager.master()) {
+ if (serverManager.selfIsMaster()) {
scheduler.scheduleTasksOnMaster();
if (!this.enableRoleElected && !serverManager.onlySingleNode()) {
// assigned-Master + non-single-node don't need to execute tasks
@@ -347,10 +347,10 @@ public final class TaskManager {
}
// Execute queued tasks scheduled to current server
- scheduler.executeTasksOnWorker(serverManager.selfServerId());
+ scheduler.executeTasksOnWorker(serverManager.selfNodeId());
// Cancel tasks scheduled to current server
- scheduler.cancelTasksOnWorker(serverManager.selfServerId());
+ scheduler.cancelTasksOnWorker(serverManager.selfNodeId());
} finally {
LockUtil.unlock(graph, LockUtil.GRAPH_LOCK);
}