You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by hu...@apache.org on 2024/04/27 03:20:03 UTC
(kvrocks-controller) branch unstable updated: Refactor the implementation of the controller's probe and migration (#167)
This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/kvrocks-controller.git
The following commit(s) were added to refs/heads/unstable by this push:
new 5c2019b Refactor the implementation of the controller's probe and migration (#167)
5c2019b is described below
commit 5c2019be7e9efac28958d9629cd88735a5949d90
Author: hulk <hu...@gmail.com>
AuthorDate: Sat Apr 27 11:19:58 2024 +0800
Refactor the implementation of the controller's probe and migration (#167)
Before this PR, the API and controller is very hard to add test cases
since we don't design any interface for this purpose. And many
components are designed as an async mode by default which is not
necessary.
---
README.md | 4 -
cmd/client/client.go | 66 ----
cmd/client/completer.go | 193 ----------
cmd/client/executor.go | 298 ---------------
util/key.go => cmd/client/main.go | 8 +-
cmd/client/printer.go | 101 -----
cmd/client/prompt.go | 78 ----
cmd/client/request.go | 242 ------------
cmd/server/main.go | 5 +-
config/config.go | 24 +-
config/config.yaml | 8 +-
consts/context_key.go | 6 +-
{metadata => consts}/errors.go | 23 +-
controller/cluster.go | 342 +++++++++++++++++
controller/cluster_test.go | 234 ++++++++++++
controller/controller.go | 256 +++++++------
controller/controller_test.go | 78 ++++
controller/failover/cluster.go | 219 -----------
controller/failover/failover.go | 150 --------
controller/migrate/migrator.go | 245 ------------
controller/probe/cluster.go | 183 ---------
controller/probe/probe.go | 118 ------
controller/syncer.go | 139 -------
docs/API.md | 53 +--
docs/images/client.gif | Bin 171896 -> 0 bytes
metadata/cluster.go | 143 -------
metadata/cluster_test.go | 80 ----
metadata/node.go | 63 ----
metadata/node_test.go | 39 --
metadata/shard.go | 101 -----
scripts/docker/docker-compose.yml | 30 +-
server/api/cluster.go | 160 ++++++++
server/api/cluster_test.go | 248 +++++++++++++
.../executor_test.go => server/api/handler.go | 27 +-
server/{ => api}/namespace.go | 44 ++-
server/api/namespace_test.go | 107 ++++++
server/api/node.go | 80 ++++
server/api/node_test.go | 119 ++++++
server/api/shard.go | 131 +++++++
server/api/shard_test.go | 222 +++++++++++
server/cluster.go | 219 -----------
server/cluster_test.go | 129 -------
server/controller.go | 37 --
server/{util.go => helper/helper.go} | 25 +-
server/{ => middleware}/middleware.go | 58 ++-
server/namespace_test.go | 104 ------
server/node.go | 109 ------
server/node_test.go | 99 -----
server/route.go | 71 ++--
server/server.go | 52 ++-
server/shard.go | 260 -------------
server/shard_test.go | 101 -----
storage/failover.go | 68 ----
storage/migrate.go | 79 ----
storage/node.go | 268 --------------
storage/persistence/persistence.go | 44 ---
storage/setup_test.go | 45 ---
storage/shard.go | 204 ----------
storage/storage.go | 262 -------------
storage/storage_test.go | 104 ------
store/cluster.go | 296 +++++++++++++++
storage/helper.go => store/cluster_mock_node.go | 39 +-
store/cluster_node.go | 290 +++++++++++++++
store/cluster_node_test.go | 88 +++++
store/cluster_shard.go | 260 +++++++++++++
store/cluster_shard_test.go | 70 ++++
store/cluster_test.go | 76 ++++
store/engine/engine.go | 127 +++++++
{storage/persistence => store/engine}/etcd/etcd.go | 14 +-
.../persistence => store/engine}/etcd/etcd_test.go | 0
.../engine}/zookeeper/zookeeper.go | 10 +-
.../engine}/zookeeper/zookeeper_test.go | 0
{storage => store}/event.go | 14 +-
cmd/client/error.go => store/helper.go | 25 +-
{metadata => store}/slot.go | 6 +-
{metadata => store}/slot_test.go | 2 +-
store/store.go | 242 ++++++++++++
store/store_test.go | 97 +++++
util/redis.go | 411 ---------------------
79 files changed, 3704 insertions(+), 5368 deletions(-)
diff --git a/README.md b/README.md
index 8ebf148..07fca9b 100644
--- a/README.md
+++ b/README.md
@@ -39,8 +39,4 @@ $ ./_build/kvctl-server -c config/config.yaml
```
-### 2. Create the namespace and cluster via kvctl
-
-
-
For the HTTP API, you can find the [HTTP API(work in progress)](docs/API.md) for more details.
diff --git a/cmd/client/client.go b/cmd/client/client.go
deleted file mode 100644
index 1b6443d..0000000
--- a/cmd/client/client.go
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import (
- "flag"
- "os"
-
- "github.com/c-bata/go-prompt"
-)
-
-var config struct {
- Endpoint string
-}
-
-func init() {
- flag.StringVar(&config.Endpoint, "e", "", "set Kvrocks controller server endpoint")
-}
-
-func main() {
- flag.Parse()
-
- envVar := os.Getenv("KVROCKS_CONTROLLER_HTTP_ADDR")
- if len(envVar) > 0 {
- config.Endpoint = envVar
- }
-
- if len(config.Endpoint) == 0 {
- config.Endpoint = "http://127.0.0.1:9379"
- }
- promptCtx := NewPromptContext()
- request := NewRequest(config.Endpoint)
- completer := NewCompleter(promptCtx, request)
- executor := NewExecutor(promptCtx, request)
- executorFunc := func(input string) {
- if quit := executor.Execute(input); quit {
- os.Exit(0)
- }
- }
-
- p := prompt.New(
- executorFunc,
- completer.Complete,
- prompt.OptionPrefix(">>"),
- prompt.OptionLivePrefix(promptCtx.Prefix),
- prompt.OptionTitle("kvctl"),
- )
- p.Run()
-}
diff --git a/cmd/client/completer.go b/cmd/client/completer.go
deleted file mode 100644
index 23a2b15..0000000
--- a/cmd/client/completer.go
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import (
- "strconv"
- "strings"
- "unicode"
-
- "github.com/apache/kvrocks-controller/metadata"
-
- "github.com/c-bata/go-prompt"
-)
-
-type Completer struct {
- promptCtx *PromptContext
- request *Request
-}
-
-const (
- // Commands
- commandExit = "exit"
- commandEnter = "cd"
- commandCreate = "create"
- commandDelete = "rm"
- commandList = "ls"
-
- // Options
- optionNodes = "--nodes"
- optionReplicas = "--replicas"
- optionPassword = "--password"
-
- // Resource Types
- typeNamespace = "namespace"
- typeCluster = "cluster"
- typeShard = "shard"
-)
-
-var commands = []prompt.Suggest{
- {Text: commandExit, Description: "Exit the program"},
- {Text: commandEnter, Description: "Enter a namespace, cluster or shard"},
- {Text: commandCreate, Description: "Create a namespace, cluster or shard"},
- {Text: commandDelete, Description: "Delete a namespace, cluster or shard"},
- {Text: commandList, Description: "List namespace, cluster or shard"},
-}
-
-var options = []prompt.Suggest{
- {Text: optionReplicas, Description: "Replica count, default is 1"},
- {Text: optionNodes, Description: "Node list, separated by comma"},
- {Text: optionPassword, Description: "Node's password, all nodes should have the same password"},
-}
-
-func NewCompleter(promptCtx *PromptContext, request *Request) *Completer {
- return &Completer{
- request: request,
- promptCtx: promptCtx,
- }
-}
-
-func GetWords(text string) []string {
- return strings.Fields(text)
-}
-
-func (c *Completer) IsSpaceOrTab(d prompt.Document) bool {
- text := d.TextBeforeCursor()
- if len(text) == 0 {
- return false
- }
- return unicode.IsSpace(rune(text[len(text)-1]))
-}
-
-func (c *Completer) CompleteOptions(d prompt.Document) []prompt.Suggest {
- words := GetWords(d.TextBeforeCursor())
- lastWord := words[len(words)-1]
- return prompt.FilterContains(options, lastWord, true)
-}
-
-func (c *Completer) CompleteResourceType(d prompt.Document) []prompt.Suggest {
- words := GetWords(d.TextBeforeCursor())
- if len(words) > 3 {
- return nil
- }
- prefix := ""
- if len(words) == 2 {
- prefix = words[1]
- }
- state := c.promptCtx.state
- switch state {
-
- case promptStateRoot:
- return prompt.FilterContains([]prompt.Suggest{{Text: typeNamespace}}, prefix, true)
- case promptStateNamespace:
- return prompt.FilterContains([]prompt.Suggest{{Text: typeCluster}}, prefix, true)
- case promptStateCluster:
- return prompt.FilterContains([]prompt.Suggest{{Text: typeShard}}, prefix, true)
- default:
- return nil
- }
-}
-
-func (c *Completer) CompleteResource(d prompt.Document) []prompt.Suggest {
- var err error
- var promptTexts []string
-
- words := GetWords(d.TextBeforeCursor())
- if len(words) > 3 {
- return nil
- }
- prefix := ""
- if len(words) == 2 {
- prefix = words[1]
- }
- state := c.promptCtx.state
- if state == promptStateRoot {
- promptTexts, err = c.request.ListNamespace()
- } else if state == promptStateNamespace {
- ns := c.promptCtx.namespace
- promptTexts, err = c.request.ListCluster(ns)
- } else if state == promptStateCluster {
- ns := c.promptCtx.namespace
- cluster := c.promptCtx.cluster
- var clusterInfo *metadata.Cluster
- clusterInfo, err = c.request.GetCluster(ns, cluster)
- if clusterInfo != nil {
- for i := 0; i < len(clusterInfo.Shards); i++ {
- promptTexts = append(promptTexts, strconv.Itoa(i))
- }
- }
- } else if state == promptStateShard {
- promptTexts = []string{}
- }
- if err != nil {
- return nil
- }
-
- if state != promptStateRoot {
- promptTexts = append(promptTexts, "..")
- }
- suggestions := make([]prompt.Suggest, len(promptTexts))
- for i, text := range promptTexts {
- suggestions[i] = prompt.Suggest{Text: text}
- }
- return prompt.FilterContains(suggestions, prefix, true)
-}
-
-func (c *Completer) Complete(d prompt.Document) []prompt.Suggest {
- words := GetWords(d.TextBeforeCursor())
- wordCnt := len(words)
-
- if wordCnt == 0 {
- return commands
- }
- // Should prompt commands if the last character is space or tab.
- if wordCnt == 1 && !c.IsSpaceOrTab(d) {
- return prompt.FilterContains(commands, words[0], true)
- }
- state := c.promptCtx.state
- if state == promptStateRoot && wordCnt > 2 {
- return nil
- }
-
- command := words[0]
- switch strings.ToLower(command) {
- case commandEnter, commandDelete:
- return c.CompleteResource(d)
- case commandCreate:
- if wordCnt == 1 && c.IsSpaceOrTab(d) || wordCnt == 2 && !c.IsSpaceOrTab(d) {
- return c.CompleteResourceType(d)
- }
- if wordCnt == 3 && c.IsSpaceOrTab(d) ||
- wordCnt >= 4 && strings.HasPrefix(words[3], "-") {
- return c.CompleteOptions(d)
- }
- }
- return nil
-}
diff --git a/cmd/client/executor.go b/cmd/client/executor.go
deleted file mode 100644
index a82c165..0000000
--- a/cmd/client/executor.go
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import (
- "errors"
- "fmt"
- "strconv"
- "strings"
-
- "github.com/apache/kvrocks-controller/metadata"
-)
-
-const (
- parentDir = ".."
-)
-
-type Executor struct {
- promptCtx *PromptContext
- request *Request
-}
-
-func NewExecutor(promptCtx *PromptContext, request *Request) *Executor {
- return &Executor{
- promptCtx: promptCtx,
- request: request,
- }
-}
-
-func (e *Executor) Execute(input string) (quit bool) {
- if e.isQuit(input) {
- return true
- }
- words := GetWords(input)
- if len(words) == 0 {
- // do nothing
- return
- }
-
- var err error
- command := words[0]
- switch strings.ToLower(command) {
- case commandList:
- err = e.list(words)
- case commandCreate:
- err = e.create(words)
- case commandDelete:
- err = e.delete(words)
- case commandEnter:
- err = e.enter(words)
- default:
- err = ErrUnknownCommand
- }
- if err != nil {
- PrintError(err)
- }
- return
-}
-
-func (e *Executor) list(words []string) error {
- if len(words) > 1 {
- return ErrWrongArguments
- }
- switch e.promptCtx.state {
- case promptStateRoot:
- ns, err := e.request.ListNamespace()
- if err != nil {
- return err
- }
- PrintStrings(ns)
- case promptStateNamespace:
- ns := e.promptCtx.namespace
- clusters, err := e.request.ListCluster(ns)
- if err != nil {
- return err
- }
- PrintStrings(clusters)
- case promptStateCluster:
- ns := e.promptCtx.namespace
- cluster := e.promptCtx.cluster
- clusterInfo, err := e.request.GetCluster(ns, cluster)
- if err != nil {
- return err
- }
- PrintCluster(clusterInfo)
- case promptStateShard:
- ns := e.promptCtx.namespace
- cluster := e.promptCtx.cluster
- shardID := e.promptCtx.shard
- clusterInfo, err := e.request.GetCluster(ns, cluster)
- if err != nil {
- return err
- }
- if shardID < 0 || shardID >= len(clusterInfo.Shards) {
- return metadata.ErrIndexOutOfRange
- }
- PrintShard(&clusterInfo.Shards[shardID])
- }
- return nil
-}
-
-func (e *Executor) enter(words []string) error {
- if len(words) != 2 {
- return ErrWrongArguments
- }
- switch e.promptCtx.state {
- case promptStateRoot:
- namespace := words[1]
- if namespace == parentDir {
- return errors.New("already in root")
- }
- exists, err := e.request.IsNamespaceExists(namespace)
- if err != nil {
- return err
- }
- if !exists {
- return ErrNamespaceNotExits
- }
- e.promptCtx.SetNamespace(namespace)
- return nil
- case promptStateNamespace:
- ns := e.promptCtx.namespace
- cluster := words[1]
- if cluster == parentDir {
- e.promptCtx.Reset()
- } else {
- exists, err := e.request.IsClusterExists(ns, cluster)
- if err != nil {
- return err
- }
- if !exists {
- return ErrClusterNotExits
- }
- e.promptCtx.SetCluster(cluster)
- }
- return nil
- case promptStateCluster:
- ns := e.promptCtx.namespace
- cluster := e.promptCtx.cluster
- shard := words[1]
- if shard == parentDir {
- e.promptCtx.SetNamespace(ns)
- } else {
- shardID, err := strconv.Atoi(shard)
- if err != nil {
- return errors.New("shard id MUST be a number")
- }
- clusterInfo, err := e.request.GetCluster(ns, cluster)
- if err != nil {
- return err
- }
- if shardID < 0 || shardID >= len(clusterInfo.Shards) {
- return errors.New("shard id out of range")
- }
- e.promptCtx.SetShard(shardID)
- }
- return nil
- case promptStateShard:
- cluster := e.promptCtx.cluster
- if words[1] == parentDir {
- e.promptCtx.SetCluster(cluster)
- return nil
- }
- return errors.New("already in shard, can enter nothing")
- }
- return errors.New("unsupported enter state")
-}
-
-func parseClusterOptions(words []string) (*ClusterOptions, error) {
- if len(words) < 4 {
- return nil, ErrWrongArguments
- }
-
- clusterOptions := &ClusterOptions{
- Name: words[2],
- }
- for i := 3; i < len(words); i++ {
- switch words[i] {
- case "--nodes":
- if i+1 >= len(words) {
- return nil, fmt.Errorf("missing 'nodes'")
- }
- nodes := strings.Split(words[i+1], ",")
- clusterOptions.Nodes = nodes
- i++
- case "--replicas":
- if i+1 >= len(words) {
- return nil, fmt.Errorf("missing 'replicas'")
- }
- replicas, err := strconv.Atoi(words[i+1])
- if err != nil {
- return nil, fmt.Errorf("'replica' is NOT a number")
- }
- if replicas <= 0 {
- return nil, fmt.Errorf("'replica' should be greater than 0")
- }
- clusterOptions.Replicas = replicas
- i++
- case "--password":
- if i+1 >= len(words) {
- return nil, fmt.Errorf("missing 'password'")
- }
- clusterOptions.Password = words[i+1]
- i++
- default:
- return nil, fmt.Errorf("unknown option: %s", words[i])
- }
- }
- if len(clusterOptions.Nodes) == 0 {
- return nil, fmt.Errorf("missing 'nodes'")
- }
- if clusterOptions.Replicas == 0 {
- clusterOptions.Replicas = 1
- }
- if len(clusterOptions.Nodes)%clusterOptions.Replicas != 0 {
- return nil, fmt.Errorf("nodes count should be divisible by replicas")
- }
- return clusterOptions, nil
-}
-
-func (e *Executor) create(words []string) (err error) {
- var clusterOptions *ClusterOptions
- switch e.promptCtx.state {
- case promptStateRoot:
- if len(words) != 3 {
- return ErrWrongArguments
- }
- if words[1] != typeNamespace {
- return fmt.Errorf("cannot create '%s' in root state", words[2])
- }
- namespace := words[2]
- err = e.request.CreateNamespace(namespace)
- case promptStateNamespace:
- ns := e.promptCtx.namespace
- if len(words) < 3 {
- return ErrWrongArguments
- }
- if words[1] != typeCluster {
- return fmt.Errorf("cannot create '%s' in namespace state", words[2])
- }
- clusterOptions, err = parseClusterOptions(words)
- if err != nil {
- return err
- }
- err = e.request.CreateCluster(ns, clusterOptions)
- default:
- return errors.New("unsupported create state")
- }
- if err != nil {
- return err
- }
- PrintStatus("CREATED")
- return nil
-}
-
-func (e *Executor) delete(words []string) (err error) {
- if len(words) != 2 {
- return ErrWrongArguments
- }
-
- switch e.promptCtx.state {
- case promptStateRoot:
- namespace := words[1]
- err = e.request.DeleteNamespace(namespace)
- case promptStateNamespace:
- cluster := words[1]
- err = e.request.DeleteCluster(e.promptCtx.namespace, cluster)
- default:
- err = errors.New("unsupported delete state")
- }
- if err != nil {
- return err
- }
- PrintStatus("DELETED")
- return nil
-}
-
-func (e *Executor) isQuit(command string) bool {
- command = strings.TrimSpace(strings.ToLower(command))
- return command == "quit" || command == "exit"
-}
diff --git a/util/key.go b/cmd/client/main.go
similarity index 86%
rename from util/key.go
rename to cmd/client/main.go
index 67cccb6..bac6575 100644
--- a/util/key.go
+++ b/cmd/client/main.go
@@ -17,12 +17,8 @@
* under the License.
*
*/
-package util
-import (
- "fmt"
-)
+package main
-func BuildClusterKey(ns, cluster string) string {
- return fmt.Sprintf("%s/%s", ns, cluster)
+func main() {
}
diff --git a/cmd/client/printer.go b/cmd/client/printer.go
deleted file mode 100644
index 86f930d..0000000
--- a/cmd/client/printer.go
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import (
- "context"
- "fmt"
- "os"
- "strings"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/util"
- "github.com/jedib0t/go-pretty/v6/table"
-)
-
-func PrintError(err error) {
- fmt.Println("Error:", err)
-}
-
-func PrintStrings(strings []string) {
- for _, str := range strings {
- fmt.Println(str)
- }
-}
-
-func PrintStatus(status string) {
- fmt.Println(status)
-}
-
-func slotRangesToString(slotRanges []metadata.SlotRange) string {
- slotBuilder := strings.Builder{}
- for j, slotRange := range slotRanges {
- slotBuilder.WriteString(slotRange.String())
- if j != len(slotRanges)-1 {
- slotBuilder.WriteByte(' ')
- }
- }
- return slotBuilder.String()
-}
-
-func PrintCluster(cluster *metadata.Cluster) {
- t := table.NewWriter()
- t.SetOutputMirror(os.Stdout)
- t.AppendHeader(table.Row{"#", "Slots", "Master", "Slaves", "Import", "Migrate"})
- for i, shard := range cluster.Shards {
- var master string
- slaves := make([]string, 0)
- for _, node := range shard.Nodes {
- if node.IsMaster() {
- master = node.Addr
- } else {
- slaves = append(slaves, node.Addr)
- }
- }
- t.AppendRows([]table.Row{
- {i, slotRangesToString(shard.SlotRanges), master, slaves, shard.ImportSlot, shard.MigratingSlot},
- })
- }
- t.Render()
-}
-
-func PrintShard(shard *metadata.Shard) {
- t := table.NewWriter()
- t.SetOutputMirror(os.Stdout)
- t.AppendHeader(table.Row{"ID", "IP:Port", "Role", "Slots", "Import", "Migrate", "Status"})
- for _, node := range shard.Nodes {
- status := "Alive"
- if err := util.PingCmd(context.Background(), &node); err != nil {
- status = "Dead"
- }
- t.AppendRows([]table.Row{
- {
- node.ID,
- node.Addr,
- node.Role,
- slotRangesToString(shard.SlotRanges),
- shard.ImportSlot,
- shard.MigratingSlot,
- status,
- },
- })
- }
- t.Render()
-}
diff --git a/cmd/client/prompt.go b/cmd/client/prompt.go
deleted file mode 100644
index bba5d7a..0000000
--- a/cmd/client/prompt.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import "fmt"
-
-const (
- promptStateRoot = iota + 1
- promptStateNamespace
- promptStateCluster
- promptStateShard
-)
-
-type PromptContext struct {
- state int
- namespace string
- cluster string
- shard int
-}
-
-func NewPromptContext() *PromptContext {
- return &PromptContext{state: promptStateRoot}
-}
-
-func (ctx *PromptContext) Reset() {
- ctx.state = promptStateRoot
- ctx.namespace = ""
- ctx.cluster = ""
- ctx.shard = -1
-}
-
-func (ctx *PromptContext) SetNamespace(ns string) {
- ctx.state = promptStateNamespace
- ctx.namespace = ns
- ctx.cluster = ""
- ctx.shard = -1
-}
-
-func (ctx *PromptContext) SetCluster(cluster string) {
- ctx.state = promptStateCluster
- ctx.cluster = cluster
- ctx.shard = -1
-}
-
-func (ctx *PromptContext) SetShard(shard int) {
- ctx.state = promptStateShard
- ctx.shard = shard
-}
-
-func (ctx *PromptContext) Prefix() (string, bool) {
- switch ctx.state {
- case promptStateNamespace:
- return fmt.Sprintf("%s>> ", ctx.namespace), true
- case promptStateCluster:
- return fmt.Sprintf("%s/%s>> ", ctx.namespace, ctx.cluster), true
- case promptStateShard:
- return fmt.Sprintf("%s/%s/%d>> ", ctx.namespace, ctx.cluster, ctx.shard), true
- default:
- return ">> ", true
- }
-}
diff --git a/cmd/client/request.go b/cmd/client/request.go
deleted file mode 100644
index 8a18f6c..0000000
--- a/cmd/client/request.go
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package main
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "net/http"
- "strings"
-
- "github.com/apache/kvrocks-controller/metadata"
-
- "github.com/go-resty/resty/v2"
-)
-
-type Error struct {
- Message string `json:"message"`
-}
-
-type ClusterOptions struct {
- Name string `json:"name"`
- Nodes []string `json:"nodes"`
- Replicas int `json:"replicas"`
- Password string `json:"password"`
-}
-
-type Request struct {
- endpoint string
- restyCli *resty.Client
-}
-
-func NewRequest(endpoint string) *Request {
- endpoint = strings.TrimSuffix(endpoint, "/")
- restyCli := resty.New()
- restyCli.SetBaseURL(fmt.Sprintf("%s/api/v1", endpoint))
- return &Request{
- endpoint: endpoint,
- restyCli: restyCli,
- }
-}
-
-func (req *Request) ListNamespace() ([]string, error) {
- rsp, err := req.restyCli.R().Get("/namespaces")
- if err != nil {
- return nil, err
- }
-
- var result struct {
- Error *Error `json:"error"`
- Data struct {
- Namespaces []string `json:"namespaces"`
- } `json:"data"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return nil, err
- }
- if result.Error != nil {
- return nil, fmt.Errorf(result.Error.Message)
- }
- return result.Data.Namespaces, nil
-}
-
-func (req *Request) ListCluster(ns string) ([]string, error) {
- path := fmt.Sprintf("/namespaces/%s/clusters", ns)
- rsp, err := req.restyCli.R().Get(path)
- if err != nil {
- return nil, err
- }
- var result struct {
- Error *Error `json:"error"`
- Data struct {
- Clusters []string `json:"clusters"`
- } `json:"data"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return nil, err
- }
- if result.Error != nil {
- return nil, fmt.Errorf(result.Error.Message)
- }
- return result.Data.Clusters, nil
-}
-
-func (req *Request) CreateNamespace(ns string) error {
- rsp, err := req.restyCli.R().SetBody(
- map[string]interface{}{
- "namespace": ns,
- }).Post("/namespaces")
- if err != nil {
- return err
- }
- var result struct {
- Error *Error `json:"error"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return err
- }
- if result.Error != nil {
- return fmt.Errorf(result.Error.Message)
- }
- if rsp.StatusCode() != http.StatusCreated {
- return fmt.Errorf("create namespace %s failed: %s", ns, rsp.Status())
- }
- return nil
-}
-
-func (req *Request) CreateCluster(ns string, options *ClusterOptions) error {
- path := fmt.Sprintf("/namespaces/%s/clusters", ns)
- rsp, err := req.restyCli.R().SetBody(options).Post(path)
- if err != nil {
- return err
- }
- var result struct {
- Error *Error `json:"error"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return err
- }
- if result.Error != nil {
- return fmt.Errorf(result.Error.Message)
- }
- if rsp.StatusCode() != http.StatusCreated {
- return fmt.Errorf("create clsuter failed: %s", rsp.Status())
- }
- return nil
-}
-
-func (req *Request) IsNamespaceExists(ns string) (bool, error) {
- path := fmt.Sprintf("/namespaces/%s", ns)
- rsp, err := req.restyCli.R().Get(path)
- if err != nil {
- return false, err
- }
- var result struct {
- Error *Error `json:"error"`
- Data struct {
- Exists bool `json:"exists"`
- } `json:"data"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return false, err
- }
- if result.Error != nil {
- return false, fmt.Errorf(result.Error.Message)
- }
- return result.Data.Exists, nil
-}
-
-func (req *Request) IsClusterExists(ns, cluster string) (bool, error) {
- clusterInfo, err := req.GetCluster(ns, cluster)
- if err != nil {
- return false, err
- }
- return clusterInfo != nil, nil
-}
-
-func (req *Request) GetCluster(ns, cluster string) (*metadata.Cluster, error) {
- path := fmt.Sprintf("/namespaces/%s/clusters/%s", ns, cluster)
- rsp, err := req.restyCli.R().Get(path)
- if err != nil {
- return nil, err
- }
- var result struct {
- Error *Error `json:"error"`
- Data struct {
- Cluster *metadata.Cluster `json:"cluster"`
- } `json:"data"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return nil, err
- }
- if result.Error != nil {
- return nil, fmt.Errorf(result.Error.Message)
- }
- if rsp.StatusCode() == http.StatusNotFound {
- return nil, errors.New("cluster not found")
- }
- if rsp.StatusCode() != http.StatusOK {
- return nil, fmt.Errorf("get clsuter %s failed: %s", cluster, rsp.Status())
- }
- return result.Data.Cluster, nil
-}
-
-func (req *Request) DeleteNamespace(ns string) error {
- path := fmt.Sprintf("/namespaces/%s", ns)
- rsp, err := req.restyCli.R().Delete(path)
- if err != nil {
- return err
- }
- var result struct {
- Error *Error `json:"error"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return err
- }
- if result.Error != nil {
- return fmt.Errorf(result.Error.Message)
- }
- if rsp.IsError() {
- return fmt.Errorf("delete namespace %s failed: %s", ns, rsp.Status())
- }
- return nil
-}
-
-func (req *Request) DeleteCluster(ns, cluster string) error {
- path := fmt.Sprintf("/namespaces/%s/clusters/%s", ns, cluster)
- rsp, err := req.restyCli.R().Delete(path)
- if err != nil {
- return err
- }
- var result struct {
- Error *Error `json:"error"`
- }
- if err := json.Unmarshal(rsp.Body(), &result); err != nil {
- return err
- }
- if result.Error != nil {
- return fmt.Errorf(result.Error.Message)
- }
- if rsp.IsError() {
- return fmt.Errorf("delete cluster %s failed: %s", cluster, rsp.Status())
- }
- return nil
-}
diff --git a/cmd/server/main.go b/cmd/server/main.go
index 1eb092c..461107b 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -20,6 +20,7 @@
package main
import (
+ "context"
"flag"
"os"
"os/signal"
@@ -67,6 +68,7 @@ func main() {
// os signal handler
shutdownCh := make(chan struct{})
registerSignal(shutdownCh)
+ ctx, cancelFn := context.WithCancel(context.Background())
flag.Parse()
@@ -92,13 +94,14 @@ func main() {
logger.Get().With(zap.Error(err)).Error("Failed to create the server")
return
}
- if err := srv.Start(); err != nil {
+ if err := srv.Start(ctx); err != nil {
logger.Get().With(zap.Error(err)).Error("Failed to start the server")
return
}
// wait for the term signal
<-shutdownCh
+ cancelFn()
if err := srv.Stop(); err != nil {
logger.Get().With(zap.Error(err)).Error("Failed to close the server")
} else {
diff --git a/config/config.go b/config/config.go
index c0370df..559f092 100644
--- a/config/config.go
+++ b/config/config.go
@@ -25,8 +25,8 @@ import (
"net"
"os"
- "github.com/apache/kvrocks-controller/storage/persistence/etcd"
- "github.com/apache/kvrocks-controller/storage/persistence/zookeeper"
+ "github.com/apache/kvrocks-controller/store/engine/etcd"
+ "github.com/apache/kvrocks-controller/store/engine/zookeeper"
"github.com/go-playground/validator/v10"
)
@@ -57,6 +57,16 @@ type Config struct {
Controller *ControllerConfig `yaml:"controller"`
}
+func DefaultFailOverConfig() *FailOverConfig {
+ return &FailOverConfig{
+ GCIntervalSeconds: 3600,
+ PingIntervalSeconds: 3,
+ MaxPingCount: 5,
+ MinAliveSize: 10,
+ MaxFailureRatio: 0.6,
+ }
+}
+
func Default() *Config {
c := &Config{
Etcd: &etcd.Config{
@@ -86,16 +96,6 @@ func (c *Config) Validate() error {
return nil
}
-func DefaultFailOverConfig() *FailOverConfig {
- return &FailOverConfig{
- GCIntervalSeconds: 3600,
- PingIntervalSeconds: 3,
- MaxPingCount: 5,
- MinAliveSize: 10,
- MaxFailureRatio: 0.6,
- }
-}
-
func (c *Config) getAddr() string {
// env has higher priority than configuration.
// case: get addr from env
diff --git a/config/config.yaml b/config/config.yaml
index d09fdf4..2bec5dc 100644
--- a/config/config.yaml
+++ b/config/config.yaml
@@ -19,7 +19,7 @@
addr: "127.0.0.1:9379"
-# Which storage engine should be used by controller
+# Which store engine should be used by controller
# options: etcd, zookeeper
# default: etcd
storage_type: etcd
@@ -45,7 +45,5 @@ zookeeper:
controller:
failover:
- gc_interval_seconds: 3600
- ping_interval_seconds: 5
- min_alive_size: 10
- max_failure_ratio: 0.6
+ ping_interval_seconds: 3
+ min_alive_size: 5
\ No newline at end of file
diff --git a/consts/context_key.go b/consts/context_key.go
index d686330..f77ab01 100644
--- a/consts/context_key.go
+++ b/consts/context_key.go
@@ -20,9 +20,9 @@
package consts
const (
- ContextKeyStorage = "_context_key_storage"
- ContextKeyMigrator = "_context_key_migrate"
- ContextKeyFailover = "_context_key_failover"
+ ContextKeyStore = "_context_key_storage"
+ ContextKeyCluster = "_context_key_cluster"
+ ContextKeyClusterShard = "_context_key_cluster_shard"
)
const (
diff --git a/metadata/errors.go b/consts/errors.go
similarity index 51%
rename from metadata/errors.go
rename to consts/errors.go
index 3906cdf..a44c032 100644
--- a/metadata/errors.go
+++ b/consts/errors.go
@@ -17,16 +17,25 @@
* under the License.
*
*/
-package metadata
-import (
- "errors"
-)
+package consts
+
+import "errors"
var (
- ErrEntryNoExists = errors.New("the entry does not exist")
- ErrEntryExisted = errors.New("the entry already existed")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrNotFound = errors.New("not found")
+ ErrAlreadyExists = errors.New("already exists")
ErrIndexOutOfRange = errors.New("index out of range")
- ErrShardNoMatchPromoteNode = errors.New("no match promote node in shard")
+ ErrShardIsSame = errors.New("source and target shard is same")
+ ErrSlotOutOfRange = errors.New("slot out of range")
+ ErrSlotNotBelongToAnyShard = errors.New("slot not belong to any shard")
+ ErrNodeIsNotMaster = errors.New("the old node is not master")
+ ErrOldMasterNodeNotFound = errors.New("old master node not found")
ErrShardNoReplica = errors.New("no replica in shard")
+ ErrShardIsServicing = errors.New("shard is servicing")
+ ErrShardSlotIsMigrating = errors.New("shard slot is migrating")
+ ErrShardNoMatchNewMaster = errors.New("no match new master in shard")
+ ErrMismatchMigrateSlot = errors.New("mismatch migrate slot")
+ ErrMigrateSlotFailed = errors.New("migrate slot failed")
)
diff --git a/controller/cluster.go b/controller/cluster.go
new file mode 100644
index 0000000..b086419
--- /dev/null
+++ b/controller/cluster.go
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package controller
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "time"
+
+ "go.uber.org/zap"
+
+ "github.com/apache/kvrocks-controller/logger"
+ "github.com/apache/kvrocks-controller/store"
+)
+
+var (
+ ErrClusterNotInitialized = errors.New("ERR CLUSTERDOWN The cluster is not initialized")
+ ErrRestoringBackUp = errors.New("ERR LOADING kvrocks is restoring the db from backup")
+)
+
+type ClusterOptions struct {
+ pingInterval time.Duration
+ maxFailureCount int64
+}
+
+type ClusterChecker struct {
+ options ClusterOptions
+ clusterStore store.Store
+ clusterMu sync.Mutex
+ cluster *store.Cluster
+
+ namespace string
+ clusterName string
+
+ failureMu sync.Mutex
+ failureCounts map[string]int64
+ syncCh chan struct{}
+
+ ctx context.Context
+ cancelFn context.CancelFunc
+
+ wg sync.WaitGroup
+}
+
+func NewClusterProbe(s store.Store, ns, cluster string) *ClusterChecker {
+ ctx, cancel := context.WithCancel(context.Background())
+ c := &ClusterChecker{
+ namespace: ns,
+ clusterName: cluster,
+
+ clusterStore: s,
+ options: ClusterOptions{
+ pingInterval: time.Second * 3,
+ maxFailureCount: 5,
+ },
+ failureCounts: make(map[string]int64),
+ syncCh: make(chan struct{}, 1),
+
+ ctx: ctx,
+ cancelFn: cancel,
+ }
+ return c
+}
+
+func (c *ClusterChecker) Start() {
+ c.wg.Add(1)
+ go c.probeLoop()
+ c.wg.Add(1)
+ go c.migrationLoop()
+}
+
+func (c *ClusterChecker) WithPingInterval(interval time.Duration) *ClusterChecker {
+ c.options.pingInterval = interval
+ if c.options.pingInterval < 200*time.Millisecond {
+ c.options.pingInterval = 200 * time.Millisecond
+ }
+ return c
+}
+
+func (c *ClusterChecker) WithMaxFailureCount(count int64) *ClusterChecker {
+ c.options.maxFailureCount = count
+ if c.options.maxFailureCount < 1 {
+ c.options.maxFailureCount = 5
+ }
+ return c
+}
+
+func (c *ClusterChecker) probeNode(ctx context.Context, node store.Node) (int64, error) {
+ clusterInfo, err := node.GetClusterInfo(ctx)
+ if err != nil {
+ switch err.Error() {
+ case ErrRestoringBackUp.Error():
+ // The node is restoring from backup, just skip it
+ return -1, nil
+ case ErrClusterNotInitialized.Error():
+ return -1, ErrClusterNotInitialized
+ default:
+ return -1, err
+ }
+ }
+ return clusterInfo.CurrentEpoch, nil
+}
+
+func (c *ClusterChecker) increaseFailureCount(shardIndex int, node store.Node) int64 {
+ id := node.ID()
+ c.failureMu.Lock()
+ if _, ok := c.failureCounts[id]; !ok {
+ c.failureCounts[id] = 0
+ }
+ c.failureCounts[id] += 1
+ count := c.failureCounts[id]
+ c.failureMu.Unlock()
+
+ // don't add the node into the failover candidates if it's not a master node
+ if !node.IsMaster() {
+ return count
+ }
+
+ log := logger.Get().With(
+ zap.String("id", node.ID()),
+ zap.Bool("is_master", node.IsMaster()),
+ zap.String("addr", node.Addr()))
+ if count%c.options.maxFailureCount == 0 {
+ cluster, err := c.clusterStore.GetCluster(c.ctx, c.namespace, c.clusterName)
+ if err != nil {
+ log.Error("Failed to get the clusterName info", zap.Error(err))
+ return count
+ }
+ newMasterID, err := cluster.PromoteNewMaster(c.ctx, shardIndex, node.ID(), "")
+ if err == nil {
+ // the node is normal if it can be elected as the new master,
+ // because it requires the node is healthy.
+ c.resetFailureCount(newMasterID)
+ err = c.clusterStore.UpdateCluster(c.ctx, c.namespace, cluster)
+ }
+ if err != nil {
+ log.Error("Failed to promote the new master", zap.Error(err))
+ } else {
+ log.With(zap.String("new_master_id", newMasterID)).Info("Promote the new master")
+ }
+ }
+ return count
+}
+
+func (c *ClusterChecker) resetFailureCount(nodeID string) {
+ c.failureMu.Lock()
+ delete(c.failureCounts, nodeID)
+ c.failureMu.Unlock()
+}
+
+func (c *ClusterChecker) sendSyncEvent() {
+ select {
+ case c.syncCh <- struct{}{}:
+ case <-c.ctx.Done():
+ return
+ }
+}
+
+func (c *ClusterChecker) syncClusterToNodes(ctx context.Context) error {
+ clusterInfo, err := c.clusterStore.GetCluster(ctx, c.namespace, c.clusterName)
+ if err != nil {
+ return err
+ }
+ for _, shard := range clusterInfo.Shards {
+ for _, node := range shard.Nodes {
+ // sync the clusterName to the latest version
+ if err := node.SyncClusterInfo(ctx, clusterInfo); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (c *ClusterChecker) parallelProbeNodes(ctx context.Context, cluster *store.Cluster) {
+ for i, shard := range cluster.Shards {
+ for _, node := range shard.Nodes {
+ go func(shardIdx int, n store.Node) {
+ log := logger.Get().With(
+ zap.String("id", n.ID()),
+ zap.Bool("is_master", n.IsMaster()),
+ zap.String("addr", n.Addr()),
+ )
+ version, err := c.probeNode(ctx, n)
+ if err != nil && !errors.Is(err, ErrClusterNotInitialized) {
+ failureCount := c.increaseFailureCount(shardIdx, n)
+ log.With(zap.Error(err),
+ zap.Int64("failure_count", failureCount),
+ ).Warn("Failed to probe the node")
+ return
+ }
+ log.Debug("Probe the clusterName node")
+
+ clusterVersion := cluster.Version.Load()
+ if version < clusterVersion {
+ // sync the clusterName to the latest version
+ if err := n.SyncClusterInfo(ctx, cluster); err != nil {
+ log.With(zap.Error(err)).Error("Failed to sync the clusterName info")
+ }
+ } else if version > cluster.Version.Load() {
+ log.With(
+ zap.Int64("node.version", version),
+ zap.Int64("clusterName.version", clusterVersion),
+ ).Warn("The node is in a higher version")
+ }
+ c.resetFailureCount(n.ID())
+ }(i, node)
+ }
+ }
+}
+
+func (c *ClusterChecker) probeLoop() {
+ defer c.wg.Done()
+ log := logger.Get().With(
+ zap.String("namespace", c.namespace),
+ zap.String("clusterName", c.clusterName),
+ )
+
+ probeTicker := time.NewTicker(c.options.pingInterval)
+ defer probeTicker.Stop()
+ for {
+ select {
+ case <-probeTicker.C:
+ clusterInfo, err := c.clusterStore.GetCluster(c.ctx, c.namespace, c.clusterName)
+ if err != nil {
+ log.Error("Failed to get the clusterName info from the clusterStore", zap.Error(err))
+ break
+ }
+ c.clusterMu.Lock()
+ c.cluster = clusterInfo
+ c.clusterMu.Unlock()
+ c.parallelProbeNodes(c.ctx, clusterInfo)
+ case <-c.syncCh:
+ if err := c.syncClusterToNodes(c.ctx); err != nil {
+ log.Error("Failed to sync the clusterName to the nodes", zap.Error(err))
+ }
+ case <-c.ctx.Done():
+ return
+ }
+ }
+}
+
+func (c *ClusterChecker) tryUpdateMigrationStatus(ctx context.Context, cluster *store.Cluster) {
+ log := logger.Get().With(
+ zap.String("namespace", c.namespace),
+ zap.String("cluster", c.clusterName))
+
+ for i, shard := range cluster.Shards {
+ if !shard.IsMigrating() {
+ continue
+ }
+
+ sourceNodeClusterInfo, err := shard.GetMasterNode().GetClusterInfo(ctx)
+ if err != nil {
+ log.Error("Failed to get the cluster info from the source node", zap.Error(err))
+ return
+ }
+ if sourceNodeClusterInfo.MigratingSlot != shard.MigratingSlot {
+ log.Error("Mismatch migrate slot", zap.Int("slot", shard.MigratingSlot))
+ }
+ if shard.TargetShardIndex < 0 || shard.TargetShardIndex >= len(cluster.Shards) {
+ log.Error("Invalid target shard index", zap.Int("index", shard.TargetShardIndex))
+ }
+ targetMasterNode := cluster.Shards[shard.TargetShardIndex].GetMasterNode()
+
+ switch sourceNodeClusterInfo.MigratingState {
+ case "none", "start":
+ continue
+ case "fail":
+ c.clusterMu.Lock()
+ cluster.Shards[i].ClearMigrateState()
+ c.clusterMu.Unlock()
+ if err := c.clusterStore.SetCluster(ctx, c.namespace, cluster); err != nil {
+ log.Error("Failed to clear the migrate state", zap.Error(err))
+ }
+ log.Warn("Failed to migrate the slot", zap.Int("slot", shard.MigratingSlot))
+ case "success":
+ err := cluster.SetSlot(ctx, shard.MigratingSlot, targetMasterNode.ID())
+ if err != nil {
+ log.Error("Failed to set the slot", zap.Error(err))
+ return
+ }
+ cluster.Shards[i].SlotRanges = store.RemoveSlotRanges(cluster.Shards[i].SlotRanges,
+ []store.SlotRange{{Start: shard.MigratingSlot, Stop: shard.MigratingSlot}})
+ cluster.Shards[shard.TargetShardIndex].SlotRanges = store.MergeSlotRanges(
+ cluster.Shards[shard.TargetShardIndex].SlotRanges,
+ []store.SlotRange{{Start: shard.MigratingSlot, Stop: shard.MigratingSlot}})
+ cluster.Shards[i].ClearMigrateState()
+ if err := c.clusterStore.SetCluster(ctx, c.namespace, cluster); err != nil {
+ log.Error("Failed to update the cluster", zap.Error(err))
+ } else {
+ log.Info("Migrate the slot successfully", zap.Int("slot", shard.MigratingSlot))
+ }
+ default:
+ log.Error("Unknown migrating state", zap.String("state", sourceNodeClusterInfo.MigratingState))
+ }
+ }
+}
+
+func (c *ClusterChecker) migrationLoop() {
+ defer c.wg.Done()
+
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-c.ctx.Done():
+ return
+ case <-ticker.C:
+ c.clusterMu.Lock()
+ cluster := c.cluster
+ c.clusterMu.Unlock()
+ if cluster == nil {
+ continue
+ }
+ c.tryUpdateMigrationStatus(c.ctx, cluster)
+ }
+ }
+}
+
+func (c *ClusterChecker) Close() {
+ c.cancelFn()
+ c.wg.Wait()
+}
diff --git a/controller/cluster_test.go b/controller/cluster_test.go
new file mode 100644
index 0000000..56b2d91
--- /dev/null
+++ b/controller/cluster_test.go
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package controller
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.uber.org/atomic"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+type MockClusterStore struct {
+ *store.ClusterStore
+
+ mu sync.Mutex
+ clusters map[string]*store.Cluster
+}
+
+func NewMockClusterStore() *MockClusterStore {
+ return &MockClusterStore{
+ ClusterStore: store.NewClusterStore(engine.NewMock()),
+ clusters: make(map[string]*store.Cluster),
+ }
+}
+
+func (s *MockClusterStore) CreateCluster(ctx context.Context, ns string, cluster *store.Cluster) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.clusters[cluster.Name] = cluster
+ return nil
+}
+
+func (s *MockClusterStore) GetCluster(ctx context.Context, ns, cluster string) (*store.Cluster, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ c, ok := s.clusters[cluster]
+ if !ok {
+ return nil, consts.ErrNotFound
+ }
+ return c, nil
+}
+
+func (s *MockClusterStore) UpdateCluster(ctx context.Context, ns string, cluster *store.Cluster) error {
+ cluster.Version.Inc()
+ return s.SetCluster(ctx, ns, cluster)
+}
+
+func (s *MockClusterStore) SetCluster(ctx context.Context, ns string, cluster *store.Cluster) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.clusters[cluster.Name] = cluster
+ return nil
+}
+
+func (s *MockClusterStore) RemoveCluster(ctx context.Context, ns, cluster string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ delete(s.clusters, cluster)
+ return nil
+}
+
+func TestCluster_FailureCount(t *testing.T) {
+ ctx := context.Background()
+ ns := "test-ns"
+ clusterName := "test-clusterName"
+
+ s := NewMockClusterStore()
+ mockNode0 := store.NewClusterMockNode()
+ mockNode0.SetRole(store.RoleMaster)
+ mockNode0.Sequence = 104
+ mockNode1 := store.NewClusterMockNode()
+ mockNode1.SetRole(store.RoleSlave)
+ mockNode1.Sequence = 102
+ mockNode2 := store.NewClusterMockNode()
+ mockNode2.SetRole(store.RoleSlave)
+ mockNode2.Sequence = 103
+ mockNode3 := store.NewClusterMockNode()
+ mockNode3.SetRole(store.RoleSlave)
+ mockNode3.Sequence = 101
+
+ clusterInfo := &store.Cluster{
+ Name: clusterName,
+ Version: *atomic.NewInt64(1),
+ Shards: []*store.Shard{{
+ Nodes: []store.Node{
+ mockNode0, mockNode1, mockNode2, mockNode3,
+ },
+ SlotRanges: []store.SlotRange{{Start: 0, Stop: 16383}},
+ MigratingSlot: -1,
+ TargetShardIndex: -1,
+ }},
+ }
+ require.NoError(t, s.CreateCluster(ctx, ns, clusterInfo))
+ cluster := &ClusterChecker{
+ clusterStore: s,
+ namespace: ns,
+ clusterName: clusterName,
+ options: ClusterOptions{
+ pingInterval: time.Second,
+ maxFailureCount: 3,
+ },
+ failureCounts: make(map[string]int64),
+ syncCh: make(chan struct{}, 1),
+ }
+
+ require.EqualValues(t, 1, clusterInfo.Version.Load())
+ for i := int64(0); i < cluster.options.maxFailureCount-1; i++ {
+ require.EqualValues(t, i+1, cluster.increaseFailureCount(0, mockNode2))
+ }
+ for i := int64(0); i < cluster.options.maxFailureCount; i++ {
+ require.EqualValues(t, i+1, cluster.increaseFailureCount(0, mockNode0))
+ }
+ require.False(t, mockNode0.IsMaster())
+ // mockNode2 should become the new master since its sequence is the largest
+ require.True(t, mockNode2.IsMaster())
+ require.EqualValues(t, 2, clusterInfo.Version.Load())
+
+ require.EqualValues(t, 0, cluster.failureCounts[mockNode2.Addr()])
+ require.True(t, mockNode2.IsMaster())
+
+ // it will be always increase the failure count until the node is back again.
+ for i := int64(0); i < cluster.options.maxFailureCount*2; i++ {
+ require.EqualValues(t, i+1, cluster.increaseFailureCount(0, mockNode3))
+ }
+ cluster.resetFailureCount(mockNode3.ID())
+ require.EqualValues(t, 0, cluster.failureCounts[mockNode3.ID()])
+}
+
+func TestCluster_LoadAndProbe(t *testing.T) {
+ ctx := context.Background()
+ ns := "test-ns"
+ clusterName := "test-clusterProbe"
+ cluster, err := store.NewCluster(clusterName, []string{"127.0.0.1:7770", "127.0.0.1:7771"}, 2)
+ require.NoError(t, err)
+
+ nodes := make([]*store.ClusterNode, 0)
+ for _, shard := range cluster.Shards {
+ for _, node := range shard.Nodes {
+ clusterNode, _ := node.(*store.ClusterNode)
+ nodes = append(nodes, clusterNode)
+ }
+ }
+ require.NoError(t, cluster.Reset(ctx))
+ defer func() {
+ require.NoError(t, cluster.Reset(ctx))
+ }()
+
+ s := NewMockClusterStore()
+ require.NoError(t, s.CreateCluster(ctx, ns, cluster))
+
+ clusterProbe := NewClusterProbe(s, ns, clusterName)
+ clusterProbe.WithPingInterval(100 * time.Millisecond)
+ clusterProbe.Start()
+ defer clusterProbe.Close()
+
+ ticker := time.NewTicker(400 * time.Millisecond)
+ defer ticker.Stop()
+ <-ticker.C
+
+ for _, node := range nodes {
+ info, err := node.GetClusterInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, info.CurrentEpoch)
+ }
+ require.NoError(t, s.UpdateCluster(ctx, ns, cluster))
+
+ <-ticker.C
+ // should sync the clusterName info
+ for _, node := range nodes {
+ info, err := node.GetClusterInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 2, info.CurrentEpoch)
+ }
+ require.NoError(t, s.UpdateCluster(ctx, ns, cluster))
+ clusterProbe.sendSyncEvent()
+ <-ticker.C
+ for _, node := range nodes {
+ info, err := node.GetClusterInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 3, info.CurrentEpoch)
+ }
+}
+
+func TestCluster_MigrateSlot(t *testing.T) {
+ ctx := context.Background()
+ ns := "test-ns"
+ clusterName := "test-clusterProbe"
+ cluster, err := store.NewCluster(clusterName, []string{"127.0.0.1:7770", "127.0.0.1:7771"}, 1)
+ require.NoError(t, err)
+
+ require.NoError(t, cluster.Reset(ctx))
+ require.NoError(t, cluster.SyncToNodes(ctx))
+ defer func() {
+ require.NoError(t, cluster.Reset(ctx))
+ }()
+ require.NoError(t, cluster.MigrateSlot(ctx, 0, 1, false))
+
+ s := NewMockClusterStore()
+ require.NoError(t, s.CreateCluster(ctx, ns, cluster))
+
+ clusterProbe := NewClusterProbe(s, ns, clusterName)
+ clusterProbe.WithPingInterval(100 * time.Millisecond)
+ clusterProbe.Start()
+ defer clusterProbe.Close()
+
+ ticker := time.NewTicker(400 * time.Millisecond)
+ defer ticker.Stop()
+ <-ticker.C
+}
diff --git a/controller/controller.go b/controller/controller.go
index 3f7d81f..4e6ec99 100644
--- a/controller/controller.go
+++ b/controller/controller.go
@@ -22,163 +22,213 @@ package controller
import (
"context"
"fmt"
- "os"
"sync"
+ "time"
"go.uber.org/atomic"
"go.uber.org/zap"
"github.com/apache/kvrocks-controller/config"
- "github.com/apache/kvrocks-controller/controller/failover"
- "github.com/apache/kvrocks-controller/controller/migrate"
- "github.com/apache/kvrocks-controller/controller/probe"
+ "github.com/apache/kvrocks-controller/consts"
"github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
+ "github.com/apache/kvrocks-controller/store"
+)
+
+const (
+ stateInit = iota + 1
+ stateRunning
+ stateClosed
)
type Controller struct {
- storage *storage.Storage
- probe *probe.Probe
- failover *failover.FailOver
- migrator *migrate.Migrator
+ config *config.ControllerConfig
+ clusterStore *store.ClusterStore
mu sync.Mutex
- syncers map[string]*Syncer
- isLoaded atomic.Bool
+ clusters map[string]*ClusterChecker
- stopCh chan struct{}
- closeOnce sync.Once
+ wg sync.WaitGroup
+ state atomic.Int32
+ readyCh chan struct{}
+ closeCh chan struct{}
}
-func New(s *storage.Storage, config *config.Config) (*Controller, error) {
- failover := failover.New(s, config.Controller.FailOver)
- return &Controller{
- storage: s,
- failover: failover,
- migrator: migrate.New(s),
- probe: probe.New(s, failover),
- syncers: make(map[string]*Syncer, 0),
- stopCh: make(chan struct{}),
- }, nil
+func New(s *store.ClusterStore, config *config.ControllerConfig) (*Controller, error) {
+ c := &Controller{
+ config: config,
+ clusterStore: s,
+ clusters: make(map[string]*ClusterChecker),
+ readyCh: make(chan struct{}, 1),
+ closeCh: make(chan struct{}),
+ }
+ c.state.Store(stateInit)
+ return c, nil
}
-func (c *Controller) Start() error {
- go c.syncLoop()
+func (c *Controller) Start(ctx context.Context) error {
+ if !c.state.CAS(stateInit, stateRunning) {
+ return nil
+ }
+
+ c.wg.Add(1)
+ go c.syncLoop(ctx)
return nil
}
-func (c *Controller) loadModules() error {
- if c.isLoaded.CAS(false, true) {
- ctx := context.Background()
- if err := c.failover.Load(); err != nil {
- return fmt.Errorf("load failover module: %w", err)
- }
- if err := c.probe.Load(ctx); err != nil {
- return fmt.Errorf("load probe module: %w", err)
+func (c *Controller) WaitReady() {
+ <-c.readyCh
+}
+
+// suspend stops the controller from processing events if it's not the leader
+func (c *Controller) suspend() {
+ c.mu.Lock()
+ for key, cluster := range c.clusters {
+ cluster.Close()
+ delete(c.clusters, key)
+ }
+ c.mu.Unlock()
+}
+
+// resume starts the controller to process events
+func (c *Controller) resume(ctx context.Context) error {
+ namespaces, err := c.clusterStore.ListNamespace(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to list namespaces: %w", err)
+ }
+ for _, ns := range namespaces {
+ clusters, err := c.clusterStore.ListCluster(ctx, ns)
+ if err != nil {
+ return fmt.Errorf("failed to list clusters: %w", err)
}
- if err := c.migrator.Load(ctx); err != nil {
- return fmt.Errorf("load migration module: %w", err)
+ for _, cluster := range clusters {
+ c.addCluster(ns, cluster)
}
}
return nil
}
-func (c *Controller) unloadModules() {
- if c.isLoaded.CAS(true, false) {
- c.probe.Shutdown()
- c.failover.Shutdown()
- c.migrator.Shutdown()
- c.isLoaded.Store(false)
+func (c *Controller) becomeLeader(ctx context.Context, prevTermLeader string) {
+ if prevTermLeader == c.clusterStore.ID() {
+ return
+ }
+ if err := c.resume(ctx); err != nil {
+ logger.Get().Error("Failed to resume the controller", zap.Error(err))
+ return
}
+ logger.Get().Info("Became the leader, resume the controller")
}
-func (c *Controller) syncLoop() {
+func (c *Controller) syncLoop(ctx context.Context) {
+ defer c.wg.Done()
+
prevTermLeader := ""
go c.leaderEventLoop()
+ if c.clusterStore.IsLeader() {
+ c.becomeLeader(ctx, prevTermLeader)
+ }
+
+ c.readyCh <- struct{}{}
for {
select {
- case <-c.storage.LeaderChange():
- if c.storage.IsLeader() {
- if err := c.loadModules(); err != nil {
- logger.Get().With(zap.Error(err)).Error("Failed to load module, will exit")
- os.Exit(1)
- }
- currentTermLeader := c.storage.Leader()
- if prevTermLeader == "" {
- logger.Get().Info("Start as the leader")
- } else if prevTermLeader != currentTermLeader {
- logger.Get().With(
- zap.String("prev", prevTermLeader),
- ).Info("Become the leader")
- }
- prevTermLeader = currentTermLeader
+ case <-c.clusterStore.LeaderChange():
+ if c.clusterStore.IsLeader() {
+ c.becomeLeader(ctx, prevTermLeader)
+ prevTermLeader = c.clusterStore.ID()
} else {
- c.unloadModules()
- logger.Get().Info("Lost the leader campaign, will unload modules")
+ if prevTermLeader != c.clusterStore.ID() {
+ continue
+ }
+ c.suspend()
+ logger.Get().Warn("Lost the leader, suspend the controller")
}
- case <-c.stopCh:
+ case <-c.closeCh:
return
}
}
}
-func (c *Controller) handleEvent(event *storage.Event) {
- if event.Namespace == "" || event.Cluster == "" {
- return
- }
- key := util.BuildClusterKey(event.Namespace, event.Cluster)
- c.mu.Lock()
- if _, ok := c.syncers[key]; !ok {
- c.syncers[key] = NewSyncer(c.storage)
- }
- syncer := c.syncers[key]
- c.mu.Unlock()
-
- syncer.Notify(event)
-}
-
func (c *Controller) leaderEventLoop() {
for {
select {
- case event := <-c.storage.Notify():
- if !c.storage.IsLeader() {
+ case event := <-c.clusterStore.Notify():
+ if !c.clusterStore.IsLeader() || event.Type != store.EventCluster {
continue
}
- c.handleEvent(&event)
- switch event.Type { // nolint
- case storage.EventCluster:
- switch event.Command {
- case storage.CommandCreate:
- c.probe.AddCluster(event.Namespace, event.Cluster)
- case storage.CommandRemove:
- c.probe.RemoveCluster(event.Namespace, event.Cluster)
- default:
- }
+ switch event.Command {
+ case store.CommandCreate:
+ c.addCluster(event.Namespace, event.Cluster)
+ case store.CommandRemove:
+ c.removeCluster(event.Namespace, event.Cluster)
+ case store.CommandUpdate:
+ c.updateCluster(event.Namespace, event.Cluster)
default:
+ logger.Get().Error("Unknown command", zap.Any("event", event))
}
- case <-c.stopCh:
+ case <-c.closeCh:
return
}
}
}
-func (c *Controller) GetFailOver() *failover.FailOver {
- return c.failover
+func (c *Controller) buildClusterKey(namespace, clusterName string) string {
+ return namespace + "/" + clusterName
+}
+
+func (c *Controller) addCluster(namespace, clusterName string) {
+ key := c.buildClusterKey(namespace, clusterName)
+ cluster := NewClusterProbe(c.clusterStore, namespace, clusterName).
+ WithPingInterval(time.Duration(c.config.FailOver.PingIntervalSeconds) * time.Second).
+ WithMaxFailureCount(c.config.FailOver.MaxPingCount)
+ cluster.Start()
+
+ c.mu.Lock()
+ c.clusters[key] = cluster
+ c.mu.Unlock()
}
-func (c *Controller) GetMigrate() *migrate.Migrator {
- return c.migrator
+func (c *Controller) getCluster(namespace, clusterName string) (*ClusterChecker, error) {
+ key := c.buildClusterKey(namespace, clusterName)
+ c.mu.Lock()
+ cluster, ok := c.clusters[key]
+ c.mu.Unlock()
+ if !ok {
+ return nil, consts.ErrNotFound
+ }
+ return cluster, nil
}
-func (c *Controller) Stop() error {
- c.closeOnce.Do(func() {
- for _, syncer := range c.syncers {
- syncer.Close()
- }
- close(c.stopCh)
- util.CloseRedisClients()
- c.unloadModules()
- })
- return nil
+func (c *Controller) removeCluster(namespace, clusterName string) {
+ key := c.buildClusterKey(namespace, clusterName)
+ c.mu.Lock()
+ if cluster, ok := c.clusters[key]; ok {
+ cluster.Close()
+ }
+ delete(c.clusters, key)
+ c.mu.Unlock()
+}
+
+func (c *Controller) updateCluster(namespace, clusterName string) {
+ key := c.buildClusterKey(namespace, clusterName)
+ c.mu.Lock()
+ cluster, ok := c.clusters[key]
+ c.mu.Unlock()
+ if !ok {
+ logger.Get().With(
+ zap.String("namespace", namespace),
+ zap.String("clusterName", clusterName),
+ ).Error("Cluster not found")
+ return
+ }
+ cluster.sendSyncEvent()
+}
+
+func (c *Controller) Close() {
+ if !c.state.CAS(stateRunning, stateClosed) {
+ return
+ }
+
+ c.suspend()
+ close(c.readyCh)
+ close(c.closeCh)
+ c.wg.Wait()
}
diff --git a/controller/controller_test.go b/controller/controller_test.go
new file mode 100644
index 0000000..a0cc862
--- /dev/null
+++ b/controller/controller_test.go
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package controller
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/kvrocks-controller/config"
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+func TestController_Basics(t *testing.T) {
+ ctx := context.Background()
+ ns := "test-ns"
+ cluster0, err := store.NewCluster("test-cluster-0", []string{"127.0.0.1:7770"}, 1)
+ require.NoError(t, err)
+ cluster1, err := store.NewCluster("test-cluster-1", []string{"127.0.0.1:7771"}, 1)
+ require.NoError(t, err)
+
+ s := store.NewClusterStore(engine.NewMock())
+ require.True(t, s.IsLeader())
+ require.NoError(t, s.CreateCluster(ctx, ns, cluster0))
+ require.NoError(t, s.CreateCluster(ctx, ns, cluster1))
+
+ c, err := New(s, &config.ControllerConfig{
+ FailOver: &config.FailOverConfig{
+ PingIntervalSeconds: 1,
+ },
+ })
+ require.NoError(t, err)
+ require.NoError(t, c.Start(ctx))
+ defer func() {
+ c.Close()
+ }()
+
+ c.WaitReady()
+
+ t.Run("get cluster", func(t *testing.T) {
+ cluster, err := c.getCluster(ns, "test-cluster-0")
+ require.NoError(t, err)
+ require.NotNil(t, cluster)
+ cluster, err = c.getCluster(ns, "test-cluster-1")
+ require.NoError(t, err)
+ require.NotNil(t, cluster)
+ _, err = c.getCluster(ns, "test-cluster-2")
+ require.ErrorIs(t, err, consts.ErrNotFound)
+ require.NotNil(t, cluster)
+ })
+
+ t.Run("remove cluster", func(t *testing.T) {
+ c.removeCluster(ns, "test-cluster-1")
+ _, err = c.getCluster(ns, "test-cluster-1")
+ require.ErrorIs(t, err, consts.ErrNotFound)
+ })
+}
diff --git a/controller/failover/cluster.go b/controller/failover/cluster.go
deleted file mode 100644
index 2100e07..0000000
--- a/controller/failover/cluster.go
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package failover
-
-import (
- "sync"
- "time"
-
- "go.uber.org/atomic"
- "golang.org/x/net/context"
-
- "go.uber.org/zap"
-
- "github.com/apache/kvrocks-controller/config"
- "github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
-)
-
-type ClusterConfig struct {
- PingInterval int
- MaxPingCount int
- MinAliveSize int
- MaxFailureRatio float64
-}
-
-type Cluster struct {
- namespace string
- cluster string
- config *config.FailOverConfig
- storage *storage.Storage
- tasks map[string]*storage.FailoverTask
- tasksIdx []string
-
- closed atomic.Bool
- ctx context.Context
- cancelFn context.CancelFunc
-
- rw sync.RWMutex
- wg sync.WaitGroup
-}
-
-// NewCluster return a Cluster instance and start schedule goroutine
-func NewCluster(ns, cluster string, stor *storage.Storage, cfg *config.FailOverConfig) *Cluster {
- ctx, cancelFn := context.WithCancel(context.Background())
- c := &Cluster{
- namespace: ns,
- cluster: cluster,
- storage: stor,
- tasks: make(map[string]*storage.FailoverTask),
- config: cfg,
- ctx: ctx,
- cancelFn: cancelFn,
- }
-
- c.wg.Add(1)
- go c.loop()
- return c
-}
-
-// Close will release the resource when closing
-func (c *Cluster) Close() {
- if !c.closed.CAS(false, true) {
- return
- }
- c.cancelFn()
-}
-
-func (c *Cluster) AddTask(task *storage.FailoverTask) error {
- c.rw.Lock()
- defer c.rw.Unlock()
- if task == nil {
- return nil
- }
- if _, ok := c.tasks[task.Node.Addr]; ok {
- return nil
- }
- task.Status = TaskQueued
- c.tasks[task.Node.Addr] = task
- c.tasksIdx = append(c.tasksIdx, task.Node.Addr)
- return nil
-}
-
-func (c *Cluster) RemoveNodeTask(addr string) {
- c.rw.Lock()
- defer c.rw.Unlock()
- if _, ok := c.tasks[addr]; !ok {
- return
- }
- targetIndex := -1
- for i, nodeAddr := range c.tasksIdx {
- if addr == nodeAddr {
- targetIndex = i
- break
- }
- }
- c.removeTask(targetIndex)
-}
-
-func (c *Cluster) GetTasks() ([]*storage.FailoverTask, error) {
- c.rw.RLock()
- defer c.rw.RUnlock()
- var tasks []*storage.FailoverTask
- for _, task := range c.tasks {
- tasks = append(tasks, task)
- }
- return tasks, nil
-}
-
-// IsEmpty return an indicator whether the tasks queue has tasks, callend gcClusters
-func (c *Cluster) IsEmpty() bool {
- c.rw.Lock()
- defer c.rw.Unlock()
- return len(c.tasksIdx) == 0
-}
-
-// removeTask is not goroutine safety, assgin caller hold mutex
-func (c *Cluster) removeTask(idx int) {
- if idx < 0 || idx >= len(c.tasksIdx) {
- return
- }
- node := c.tasksIdx[idx]
- c.tasksIdx = append(c.tasksIdx[:idx], c.tasksIdx[idx+1:]...)
- delete(c.tasks, node)
-}
-
-func (c *Cluster) purgeTasks() {
- c.rw.Lock()
- defer c.rw.Unlock()
- for node := range c.tasks {
- delete(c.tasks, node)
- }
- c.tasksIdx = c.tasksIdx[0:0]
- return
-}
-
-func (c *Cluster) loop() {
- defer c.wg.Done()
-
- ticker := time.NewTicker(time.Duration(c.config.PingIntervalSeconds) * time.Second)
- defer ticker.Stop()
- for {
- select {
- case <-c.ctx.Done():
- return
- case <-ticker.C:
- c.rw.RLock()
- nodesCount, err := c.storage.ClusterNodesCounts(c.ctx, c.namespace, c.cluster)
- if err != nil {
- c.rw.RUnlock()
- break
- }
- if nodesCount > c.config.MinAliveSize && float64(len(c.tasks))/float64(nodesCount) > c.config.MaxFailureRatio {
- logger.Get().Sugar().Warnf("safe mode, loop ratio %.2f, allnodes: %d, failnodes: %d",
- c.config.MaxFailureRatio, nodesCount, len(c.tasks),
- )
- c.purgeTasks()
- c.rw.RUnlock()
- break
- }
- for idx, nodeAddr := range c.tasksIdx {
- if _, ok := c.tasks[nodeAddr]; !ok {
- continue
- }
- task := c.tasks[nodeAddr]
- c.removeTask(idx)
- if task.Type == ManualType {
- c.promoteMaster(c.ctx, task)
- continue
- }
- if err := util.PingCmd(c.ctx, &task.Node); err == nil {
- continue
- }
- c.promoteMaster(c.ctx, task)
- }
- c.rw.RUnlock()
- }
- }
-}
-
-func (c *Cluster) promoteMaster(ctx context.Context, task *storage.FailoverTask) {
- task.Status = TaskStarted
- task.StartTime = time.Now().Unix()
- var err error
- if task.Node.Role == metadata.RoleMaster {
- err = c.storage.PromoteNewMaster(ctx, c.namespace, c.cluster, task.ShardIdx, task.Node.ID)
- }
- if err != nil {
- task.Status = TaskFailed
- task.Err = err.Error()
- logger.Get().With(
- zap.Error(err),
- zap.Any("task", task),
- ).Error("Abort the fail over task")
- } else {
- task.Status = TaskSuccess
- logger.Get().With(zap.Any("task", task)).Info("Finish the fail over task")
- }
-
- task.FinishTime = time.Now().Unix()
-}
diff --git a/controller/failover/failover.go b/controller/failover/failover.go
deleted file mode 100644
index aa06d33..0000000
--- a/controller/failover/failover.go
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package failover
-
-import (
- "context"
- "errors"
- "sync"
- "time"
-
- "github.com/apache/kvrocks-controller/config"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
-)
-
-const (
- TaskQueued = iota + 1
- TaskStarted
- TaskSuccess
- TaskFailed
-)
-
-const (
- AutoType = iota + 1
- ManualType
-)
-
-type FailOver struct {
- storage *storage.Storage
- config *config.FailOverConfig
- clusters map[string]*Cluster
-
- ready bool
- quitCh chan struct{}
- rw sync.RWMutex
-}
-
-func New(storage *storage.Storage, failOverConfig *config.FailOverConfig) *FailOver {
- f := &FailOver{
- storage: storage,
- config: failOverConfig,
- clusters: make(map[string]*Cluster),
- quitCh: make(chan struct{}),
- }
- go f.gcClusters()
- return f
-}
-
-func (f *FailOver) Load() error {
- f.rw.Lock()
- defer f.rw.Unlock()
- f.ready = true
- return nil
-}
-
-func (f *FailOver) Shutdown() {
- f.rw.Lock()
- defer f.rw.Unlock()
- if !f.ready {
- return
- }
- f.ready = false
- for _, cluster := range f.clusters {
- cluster.Close()
- }
-}
-
-func (f *FailOver) gcClusters() {
- gcTicker := time.NewTicker(time.Duration(f.config.GCIntervalSeconds) * time.Second)
- defer gcTicker.Stop()
- for {
- select {
- case <-gcTicker.C:
- f.rw.Lock()
- for name, cluster := range f.clusters {
- if cluster.IsEmpty() {
- cluster.Close()
- delete(f.clusters, name)
- }
- }
- f.rw.Unlock()
- case <-f.quitCh:
- return
- }
- }
-}
-
-func (f *FailOver) AddNode(ns, cluster string, shardIdx int, node metadata.NodeInfo, typ int) error {
- task := &storage.FailoverTask{
- Namespace: ns,
- Cluster: cluster,
- ShardIdx: shardIdx,
- Node: node,
- Type: typ,
- Status: TaskQueued,
- QueuedTime: time.Now().Unix(),
- }
- return f.AddNodeTask(task)
-}
-
-func (f *FailOver) AddNodeTask(task *storage.FailoverTask) error {
- f.rw.Lock()
- defer f.rw.Unlock()
- if !f.ready {
- return errors.New("the fail over module is not ready")
- }
- clusterKey := util.BuildClusterKey(task.Namespace, task.Cluster)
- if _, ok := f.clusters[clusterKey]; !ok {
- f.clusters[clusterKey] = NewCluster(task.Namespace, task.Cluster, f.storage, f.config)
- }
- cluster := f.clusters[clusterKey]
- return cluster.AddTask(task)
-}
-
-func (f *FailOver) GetTasks(ctx context.Context, ns, cluster string, queryType string) ([]*storage.FailoverTask, error) {
- switch queryType {
- case "pending":
- f.rw.RLock()
- defer f.rw.RUnlock()
- clusterKey := util.BuildClusterKey(ns, cluster)
- if _, ok := f.clusters[clusterKey]; !ok {
- return nil, nil
- }
- return f.clusters[clusterKey].GetTasks()
- default:
- return nil, errors.New("unknown query type")
- }
-}
-
-func (f *FailOver) Config() *config.FailOverConfig {
- return f.config
-}
diff --git a/controller/migrate/migrator.go b/controller/migrate/migrator.go
deleted file mode 100644
index 854285e..0000000
--- a/controller/migrate/migrator.go
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package migrate
-
-import (
- "context"
- "errors"
- "fmt"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
- "go.uber.org/zap"
-)
-
-var (
- ErrMismatchMigrateSlot = errors.New("mismatched slot")
- ErrWrongSlotNode = errors.New("the slot isn't belong to the node")
- ErrSlotFailed = errors.New("migrate slot fail")
- ErrSlotCompleted = errors.New("task has been completed")
- ErrAbortedMigrateRoutine = errors.New("aborted migrate routine")
- ErrGetClusterInfo = errors.New("get cluster info failed")
-)
-
-const (
- SlotFailed = "failed"
- SlotSuccess = "success"
-)
-
-const (
- TaskStatusSuccess = iota + 1
- TaskStatusFailed
-)
-
-type Migrator struct {
- migratingTasks sync.Map
- storage *storage.Storage
-
- wg sync.WaitGroup
- shutdownCh chan struct{}
-}
-
-func New(stor *storage.Storage) *Migrator {
- m := &Migrator{
- storage: stor,
- shutdownCh: make(chan struct{}),
- }
- m.wg.Add(1)
- go func() {
- defer m.wg.Done()
- m.loop()
- }()
- return m
-}
-
-func (m *Migrator) Shutdown() {
- close(m.shutdownCh)
- m.wg.Wait()
- m.migratingTasks.Range(func(key, value interface{}) bool {
- m.migratingTasks.Delete(key)
- return true
- })
-}
-
-func (m *Migrator) Load(ctx context.Context) error {
- m.wg.Add(1)
- go func() {
- defer m.wg.Done()
- m.loop()
- }()
-
- namespaces, err := m.storage.ListNamespace(ctx)
- if err != nil {
- return err
- }
-
- for _, namespace := range namespaces {
- clusters, err := m.storage.ListCluster(ctx, namespace)
- if err != nil {
- return err
- }
- for _, cluster := range clusters {
- task, err := m.storage.GetMigratingTask(ctx, namespace, cluster)
- if err != nil {
- return err
- }
- if task == nil {
- continue
- }
- m.migratingTasks.Store(util.BuildClusterKey(namespace, cluster), task)
- }
- }
- return nil
-}
-
-func (m *Migrator) AddTask(ctx context.Context, task *storage.MigrationTask) error {
- migratingTask, err := m.storage.GetMigratingTask(ctx, task.Namespace, task.Cluster)
- if err != nil {
- return err
- }
- if migratingTask != nil {
- return errors.New("there is a migration task running")
- }
-
- sourceNode, err := m.storage.GetMasterNode(ctx, task.Namespace, task.Cluster, task.Source)
- if err != nil {
- return err
- }
- targetNode, err := m.storage.GetMasterNode(ctx, task.Namespace, task.Cluster, task.Target)
- if err != nil {
- return err
- }
- // Check if this slot is belong to the source shard
- exists, err := m.storage.HasSlot(ctx, task.Namespace, task.Cluster, task.Source, task.Slot)
- if err != nil {
- return err
- }
- if !exists {
- return ErrWrongSlotNode
- }
- if err := m.sendMigrateCommand(ctx, &sourceNode, &targetNode, task.Slot); err != nil {
- if strings.Contains(err.Error(), ErrSlotCompleted.Error()) {
- // The migration process has been finished,
- // we can continue to migrate the next slot
- return nil
- }
- return err
- }
- task.SourceNode = &sourceNode
- task.TargetNode = &targetNode
- m.migratingTasks.Store(util.BuildClusterKey(task.Namespace, task.Cluster), task)
- return m.storage.AddMigratingTask(ctx, task)
-}
-
-func (m *Migrator) loop() {
- ticker := time.NewTicker(1 * time.Second)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- ctx := context.Background()
- m.migratingTasks.Range(func(key, value interface{}) bool {
- task, _ := value.(*storage.MigrationTask)
- select {
- case <-m.shutdownCh:
- return false
- default:
- }
- if err := m.checkMigrateStatus(ctx, task); err != nil {
- m.abortMigratingTask(ctx, task, err)
- return true
- }
- err := m.storage.UpdateMigrateSlotInfo(ctx, task.Namespace, task.Cluster, task.Source, task.Target, []metadata.SlotRange{
- {Start: task.Slot, Stop: task.Slot},
- })
- if err != nil {
- logger.Get().Error("update migrate slot info failed", zap.Error(err))
- return true
- }
- m.finishMigratingTask(ctx, task)
- return true
- })
-
- case <-m.shutdownCh:
- return
- }
- }
-}
-
-func (m *Migrator) sendMigrateCommand(ctx context.Context, sourceNode, targetNode *metadata.NodeInfo, slot int) error {
- redisCli, err := util.GetRedisClient(ctx, sourceNode)
- if err != nil {
- return err
- }
- return redisCli.Do(ctx, "CLUSTERX", "migrate", strconv.Itoa(slot), targetNode.ID).Err()
-}
-
-func (m *Migrator) checkMigrateStatus(ctx context.Context, task *storage.MigrationTask) error {
- clusterInfo, err := util.ClusterInfoCmd(ctx, task.SourceNode)
- if err != nil {
- return ErrGetClusterInfo
- }
- if clusterInfo.MigratingSlot != task.Slot {
- return ErrMismatchMigrateSlot
- }
- switch clusterInfo.MigratingState {
- case SlotFailed:
- return ErrSlotFailed
- case SlotSuccess:
- return nil
- default:
- return fmt.Errorf("unknown migrate state: %s", clusterInfo.MigratingState)
- }
-}
-
-func (m *Migrator) removeMigratingTask(ctx context.Context, task *storage.MigrationTask) error {
- task.FinishTime = time.Now().Unix()
- if err := m.storage.RemoveMigratingTask(ctx, task.Namespace, task.Cluster); err != nil {
- return err
- }
- m.migratingTasks.Delete(util.BuildClusterKey(task.Namespace, task.Cluster))
- return nil
-}
-
-func (m *Migrator) abortMigratingTask(ctx context.Context, task *storage.MigrationTask, err error) {
- task.Status = TaskStatusFailed
- task.ErrorDetail = err.Error()
- task.FinishTime = time.Now().Unix()
- _ = m.removeMigratingTask(ctx, task)
- logger.Get().With(
- zap.Error(err),
- zap.Any("task", task),
- ).Error("Aborted the migrate task")
-}
-
-func (m *Migrator) finishMigratingTask(ctx context.Context, task *storage.MigrationTask) {
- task.Status = TaskStatusSuccess
- task.FinishTime = time.Now().Unix()
- _ = m.removeMigratingTask(ctx, task)
- logger.Get().With(
- zap.Any("task", task),
- ).Info("Success to migrate the slot")
-}
diff --git a/controller/probe/cluster.go b/controller/probe/cluster.go
deleted file mode 100644
index c35421e..0000000
--- a/controller/probe/cluster.go
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package probe
-
-import (
- "context"
- "errors"
- "sync"
- "time"
-
- "github.com/apache/kvrocks-controller/controller/failover"
- "github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
- "go.uber.org/zap"
-)
-
-var (
- ErrClusterNotInitialized = errors.New("ERR CLUSTERDOWN The cluster is not initialized")
- ErrRestoringBackUp = errors.New("ERR LOADING kvrocks is restoring the db from backup")
-)
-
-type Cluster struct {
- namespace string
- cluster string
- storage *storage.Storage
- failOver *failover.FailOver
- failureMu sync.Mutex
- failureCounts map[string]int64
- stopCh chan struct{}
-}
-
-func NewCluster(ns, cluster string, storage *storage.Storage, failOver *failover.FailOver) *Cluster {
- return &Cluster{
- namespace: ns,
- cluster: cluster,
- storage: storage,
- failOver: failOver,
- failureCounts: make(map[string]int64),
- stopCh: make(chan struct{}),
- }
-}
-
-func (c *Cluster) start() {
- go c.loop()
-}
-
-func (c *Cluster) probeNode(ctx context.Context, node *metadata.NodeInfo) (int64, error) {
- info, err := util.ClusterInfoCmd(ctx, node)
- if err != nil {
- switch err.Error() {
- case ErrRestoringBackUp.Error():
- // The node is restoring from backup, just skip it
- return -1, nil
- case ErrClusterNotInitialized.Error():
- return -1, ErrClusterNotInitialized
- default:
- return -1, err
- }
- }
- return info.ClusterCurrentEpoch, nil
-}
-
-func (c *Cluster) increaseFailureCount(index int, node *metadata.NodeInfo) int64 {
- log := logger.Get().With(
- zap.String("id", node.ID),
- zap.String("role", node.Role),
- zap.String("addr", node.Addr),
- )
-
- c.failureMu.Lock()
- if _, ok := c.failureCounts[node.Addr]; !ok {
- c.failureCounts[node.Addr] = 0
- }
- c.failureCounts[node.Addr] += 1
- count := c.failureCounts[node.Addr]
- c.failureMu.Unlock()
-
- /// don't add the node into the failover candidates if it's not a master node
- if node.Role != metadata.RoleMaster {
- return count
- }
-
- if count%c.failOver.Config().MaxPingCount == 0 {
- err := c.failOver.AddNode(c.namespace, c.cluster, index, *node, failover.AutoType)
- if err != nil {
- log.With(zap.Error(err)).Warn("Failed to add the node into the fail over candidates")
- } else {
- log.With(zap.Int64("failure_count", count)).Info("Add the node into the fail over candidates")
- }
- }
- return count
-}
-
-func (c *Cluster) resetFailureCount(node *metadata.NodeInfo) {
- c.failureMu.Lock()
- delete(c.failureCounts, node.Addr)
- c.failureMu.Unlock()
-}
-
-func (c *Cluster) probe(ctx context.Context, cluster *metadata.Cluster) {
- for i, shard := range cluster.Shards {
- for _, node := range shard.Nodes {
- go func(shardIdx int, node metadata.NodeInfo) {
- log := logger.Get().With(
- zap.String("id", node.ID),
- zap.String("role", node.Role),
- zap.String("addr", node.Addr),
- )
- version, err := c.probeNode(ctx, &node)
- if err != nil && !errors.Is(err, ErrClusterNotInitialized) {
- failureCount := c.increaseFailureCount(shardIdx, &node)
- log.With(zap.Error(err),
- zap.Int64("failure_count", failureCount),
- ).Error("Failed to probe the node")
- return
- }
- log.Debug("Probe the cluster node")
-
- if version < cluster.Version {
- // sync the cluster to the latest version
- err := util.SyncClusterInfo2Node(ctx, &node, cluster)
- if err != nil {
- log.With(zap.Error(err)).Error("Failed to sync the cluster info")
- }
- } else if version > cluster.Version {
- log.With(
- zap.Int64("node.version", version),
- zap.Int64("cluster.version", cluster.Version),
- ).Warn("The node is in a higher version")
- }
- c.resetFailureCount(&node)
- }(i, node)
- }
- }
-}
-
-func (c *Cluster) loop() {
- log := logger.Get().With(
- zap.String("namespace", c.namespace),
- zap.String("cluster", c.cluster),
- )
- ctx := context.Background()
- probeTicker := time.NewTicker(time.Duration(c.failOver.Config().PingIntervalSeconds) * time.Second)
- defer probeTicker.Stop()
- for {
- select {
- case <-probeTicker.C:
- clusterInfo, err := c.storage.GetClusterInfo(ctx, c.namespace, c.cluster)
- if err != nil {
- log.With(
- zap.Error(err),
- ).Error("Failed to get the cluster info from the storage")
- break
- }
- c.probe(ctx, clusterInfo)
- case <-c.stopCh:
- return
- }
- }
-}
-
-func (c *Cluster) stop() {
- close(c.stopCh)
-}
diff --git a/controller/probe/probe.go b/controller/probe/probe.go
deleted file mode 100644
index afba158..0000000
--- a/controller/probe/probe.go
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package probe
-
-import (
- "context"
- "sync"
-
- "github.com/apache/kvrocks-controller/controller/failover"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
-)
-
-type Probe struct {
- storage *storage.Storage
- failOver *failover.FailOver
- probes map[string]*Cluster
- ready bool
-
- rw sync.RWMutex
- quitCh chan struct{}
-}
-
-// New return Probe contain all methods to manager loop
-func New(storage *storage.Storage, failOver *failover.FailOver) *Probe {
- hp := &Probe{
- storage: storage,
- failOver: failOver,
- probes: make(map[string]*Cluster),
- quitCh: make(chan struct{}),
- }
- return hp
-}
-
-func (p *Probe) Load(ctx context.Context) error {
- p.rw.Lock()
- defer p.rw.Unlock()
- namespaces, err := p.storage.ListNamespace(ctx)
- if err != nil {
- return err
- }
-
- probes := make(map[string]*Cluster)
- for _, namespace := range namespaces {
- clusters, err := p.storage.ListCluster(ctx, namespace)
- if err != nil {
- return err
- }
- for _, cluster := range clusters {
- probes[util.BuildClusterKey(namespace, cluster)] = NewCluster(namespace, cluster, p.storage, p.failOver)
- }
- }
- for _, probe := range probes {
- probe.start()
- }
- p.probes = probes
- p.ready = true
- return nil
-}
-
-// Shutdown all cluster loop when leader-follower switch
-func (p *Probe) Shutdown() {
- p.rw.Lock()
- defer p.rw.Unlock()
- if !p.ready {
- return
- }
- p.ready = false
- for _, probe := range p.probes {
- probe.stop()
- }
-}
-
-// AddCluster add cluster loop and start
-func (p *Probe) AddCluster(ns, cluster string) {
- p.rw.Lock()
- defer p.rw.Unlock()
- if !p.ready {
- return
- }
- if _, ok := p.probes[util.BuildClusterKey(ns, cluster)]; ok {
- return
- }
- probe := NewCluster(ns, cluster, p.storage, p.failOver)
- probe.start()
- p.probes[util.BuildClusterKey(ns, cluster)] = probe
- return
-}
-
-// RemoveCluster delete cluster loop and stop
-func (p *Probe) RemoveCluster(ns, cluster string) {
- p.rw.Lock()
- defer p.rw.Unlock()
- if _, ok := p.probes[util.BuildClusterKey(ns, cluster)]; !ok {
- return
- }
- probe := p.probes[util.BuildClusterKey(ns, cluster)]
- probe.stop()
- delete(p.probes, util.BuildClusterKey(ns, cluster))
- return
-}
diff --git a/controller/syncer.go b/controller/syncer.go
deleted file mode 100644
index 1d62be0..0000000
--- a/controller/syncer.go
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package controller
-
-import (
- "context"
- "fmt"
- "sync"
-
- "github.com/apache/kvrocks-controller/logger"
- "go.uber.org/zap"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
-)
-
-// Syncer would sync the cluster topology information
-// to cluster nodes when it's changed.
-type Syncer struct {
- storage *storage.Storage
- wg sync.WaitGroup
- shutdown chan struct{}
- notifyCh chan storage.Event
-}
-
-func NewSyncer(s *storage.Storage) *Syncer {
- syncer := &Syncer{
- storage: s,
- shutdown: make(chan struct{}, 0),
- notifyCh: make(chan storage.Event, 8),
- }
- go syncer.loop()
- return syncer
-}
-
-func (syncer *Syncer) Notify(event *storage.Event) {
- syncer.notifyCh <- *event
-}
-
-func (syncer *Syncer) handleEvent(event *storage.Event) error {
- switch event.Type {
- case storage.EventCluster, storage.EventShard, storage.EventNode:
- return syncer.handleClusterEvent(event)
- default:
- return nil
- }
-}
-
-func (syncer *Syncer) handleClusterEvent(event *storage.Event) error {
- if event.Command != storage.CommandRemove {
- cluster, err := syncer.storage.GetClusterInfo(context.Background(), event.Namespace, event.Cluster)
- if err != nil {
- return fmt.Errorf("failed to get cluster: %w", err)
- }
- return syncClusterInfoToAllNodes(context.Background(), cluster)
- }
- // TODO: Remove related cluster tasks
- return nil
-}
-
-func (syncer *Syncer) loop() {
- defer syncer.wg.Done()
- syncer.wg.Add(1)
- for {
- select {
- case event := <-syncer.notifyCh:
- if err := syncer.handleEvent(&event); err != nil {
- logger.Get().With(
- zap.Error(err),
- zap.Any("event", event),
- ).Error("Failed to handle event")
- }
- case <-syncer.shutdown:
- return
- }
- }
-}
-
-func (syncer *Syncer) Close() {
- close(syncer.shutdown)
- close(syncer.notifyCh)
- syncer.wg.Wait()
-}
-
-func syncClusterInfoToNode(ctx context.Context, node *metadata.NodeInfo, clusterSlotsStr string, version int64) error {
- cli, err := util.GetRedisClient(ctx, node)
- if err != nil {
- return fmt.Errorf("addr: %s, dail: %w", node.Addr, err)
- }
-
- err = cli.Do(ctx, "CLUSTERX", "setnodeid", node.ID).Err()
- if err != nil {
- return fmt.Errorf("addr: %s, set node id: %w", node.Addr, err)
- }
- err = cli.Do(ctx, "CLUSTERX", "setnodes", clusterSlotsStr, version).Err()
- if err != nil {
- return fmt.Errorf("addr: %s, set nodes: %w", node.Addr, err)
- }
- return nil
-}
-
-func syncClusterInfoToAllNodes(ctx context.Context, cluster *metadata.Cluster) error {
- // FIXME: should keep retry in separate routine to prevent occurring error
- // and cause update failure.
- clusterSlotsStr, err := cluster.ToSlotString()
- if err != nil {
- return err
- }
- var errs []error
- for _, shard := range cluster.Shards {
- for _, node := range shard.Nodes {
- if err := syncClusterInfoToNode(ctx, &node, clusterSlotsStr, cluster.Version); err != nil {
- errs = append(errs, err)
- }
- }
- }
- if errs != nil {
- return fmt.Errorf("%v", errs)
- }
- return nil
-}
diff --git a/docs/API.md b/docs/API.md
index 4395e0d..240ff5b 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -606,66 +606,21 @@ DELETE /api/v1/namespaces/{namespace}/clusters/{cluster}/shards/{shard}/nodes/{n
## Migration APIs
-### Migrate Slot And Data
-
+### Migrate Slot
```shell
-POST /api/v1/namespaces/{namespace}/clusters/{cluster}/shards/migration/slot_data
+POST /api/v1/namespaces/{namespace}/clusters/{cluster}/migrate
```
#### Request Body
```json
{
- "source": 0,
"target": 1,
- "slot": 123
-}
-```
-
-#### Response JSON Body
-
-* 200
-```json
-{
- "data": "ok"
+ "slot": 123,
+ "slot_only": "false"
}
```
-* 404
-```json
-{
- "error": {
- "message": "the entry does not exist"
- }
-}
-```
-
-* 5XX
-```json
-{
- "error": {
- "message": "DETAIL ERROR STRING"
- }
-}
-```
-
-### Migrate Slot Only
-
-In this case, it only migrates slot distributions between shards and the data will not be migrated.
-So you MUST ensure that the data is already migrated before you call this API.
-
-```shell
-POST /api/v1/namespaces/{namespace}/clusters/{cluster}/shards/migration/slot_only
-```
-#### Request Body
-
-```json
-{
- "source": 0,
- "target": 1,
- "slots": ["1", "3-4"]
-}
-```
#### Response JSON Body
* 200
diff --git a/docs/images/client.gif b/docs/images/client.gif
deleted file mode 100644
index c73f39f..0000000
Binary files a/docs/images/client.gif and /dev/null differ
diff --git a/metadata/cluster.go b/metadata/cluster.go
deleted file mode 100644
index ef5e902..0000000
--- a/metadata/cluster.go
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package metadata
-
-import (
- "errors"
- "fmt"
- "sort"
- "strconv"
- "strings"
-)
-
-type ClusterConfig struct {
- Name string `json:"name"`
- HeartBeatInterval uint64 `json:"heartbeat_interval"`
- HeartBeatRetries uint64 `json:"heartbeat_retries"`
-}
-
-type Cluster struct {
- Name string `json:"name"`
- Version int64 `json:"version"`
- Shards []Shard `json:"shards"`
- Config ClusterConfig `json:"config"`
-}
-
-// SetPassword will set the password for all nodes in the cluster.
-func (cluster *Cluster) SetPassword(password string) {
- for i := 0; i < len(cluster.Shards); i++ {
- for j := 0; j < len(cluster.Shards[i].Nodes); j++ {
- cluster.Shards[i].Nodes[j].Password = password
- }
- }
-}
-
-func (cluster *Cluster) CheckOverlap(slotRange *SlotRange) error {
- for idx, shard := range cluster.Shards {
- if shard.HasOverlap(slotRange) {
- return fmt.Errorf("the slot range was owned by shard: %d", idx)
- }
- }
- return nil
-}
-
-func (cluster *Cluster) ToSlotString() (string, error) {
- var builder strings.Builder
- for i, shard := range cluster.Shards {
- shardSlotsString, err := shard.ToSlotsString()
- if err != nil {
- return "", fmt.Errorf("found err at shard[%d]: %w", i, err)
- }
- builder.WriteString(shardSlotsString)
- }
- return builder.String(), nil
-}
-
-// ParseCluster will parse the cluster string into cluster topology.
-func ParseCluster(clusterStr string) (*Cluster, error) {
- if len(clusterStr) == 0 {
- return nil, errors.New("cluster nodes string error")
- }
- nodeStrings := strings.Split(clusterStr, "\n")
- if len(nodeStrings) == 0 {
- return nil, errors.New("cluster nodes string parser error")
- }
-
- var clusterVer int64 = -1
- var shards Shards
- slaveNodes := make(map[string][]NodeInfo)
- for _, nodeString := range nodeStrings {
- fields := strings.Split(nodeString, " ")
- if len(fields) < 7 {
- return nil, fmt.Errorf("require at least 7 fields, node info[%s]", nodeString)
- }
- node := NodeInfo{
- ID: fields[0],
- Addr: strings.Split(fields[1], "@")[0],
- }
-
- if strings.Contains(fields[2], ",") {
- node.Role = strings.Split(fields[2], ",")[1]
- } else {
- node.Role = fields[2]
- }
-
- var err error
- clusterVer, err = strconv.ParseInt(fields[6], 10, 64)
- if err != nil {
- return nil, fmt.Errorf("node version error, node info[%q]", nodeString)
- }
-
- if node.Role == RoleMaster {
- if len(fields) < 9 {
- return nil, fmt.Errorf("master node element less 9, node info[%q]", nodeString)
- }
- slots, err := ParseSlotRange(fields[8])
- if err != nil {
- return nil, fmt.Errorf("master node parser slot error, node info[%q]", nodeString)
- }
- shard := Shard{
- Nodes: make([]NodeInfo, 0),
- SlotRanges: make([]SlotRange, 0),
- ImportSlot: -1,
- MigratingSlot: -1,
- }
- shard.Nodes = append(shard.Nodes, node)
- shard.SlotRanges = append(shard.SlotRanges, *slots)
- shards = append(shards, shard)
- } else if node.Role == RoleSlave {
- slaveNodes[fields[3]] = append(slaveNodes[fields[3]], node)
- } else {
- return nil, fmt.Errorf("node role error, node info[%q]", nodeString)
- }
- }
- if clusterVer == -1 {
- return nil, fmt.Errorf("no cluster version, cluster info[%q]", clusterStr)
- }
- sort.Sort(shards)
- for i := 0; i < len(shards); i++ {
- masterNode := shards[i].Nodes[0]
- shards[i].Nodes = append(shards[i].Nodes, slaveNodes[masterNode.ID]...)
- }
- return &Cluster{
- Version: clusterVer,
- Shards: shards,
- }, nil
-}
diff --git a/metadata/cluster_test.go b/metadata/cluster_test.go
deleted file mode 100644
index 3c19e46..0000000
--- a/metadata/cluster_test.go
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package metadata
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestCluster_CheckOverlap(t *testing.T) {
- cluster := &Cluster{
- Shards: []Shard{
- {SlotRanges: []SlotRange{{Start: 3, Stop: 5000}}},
- {SlotRanges: []SlotRange{{Start: 5001, Stop: 10000}}},
- {SlotRanges: []SlotRange{{Start: 10001, Stop: 16383}}},
- },
- }
- require.NoError(t, cluster.CheckOverlap(&SlotRange{Start: 1, Stop: 2}))
- require.Error(t, cluster.CheckOverlap(&SlotRange{Start: 1, Stop: 3}))
-}
-
-func TestCluster_ToSlotString(t *testing.T) {
- clusterStr := "kvrockskvrockskvrockskvrockskvrocksnode1 127.0.0.1:30001@40001 myself,master - 1676818671568 1676818671569 1 connected 0-5460\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode2 127.0.0.1:30002@40002 slave kvrockskvrockskvrockskvrockskvrocksnode1 1676818671568 1676818671569 1 connected\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode3 127.0.0.1:30003@40003 master - 1676818671568 1676818671569 1 connected 5461-10992\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode5 127.0.0.1:30005@40005 master - 1676818671568 1676818671569 1 connected 10993-16383\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode6 127.0.0.1:30006@40006 slave kvrockskvrockskvrockskvrockskvrocksnode5 1676818671568 1676818671569 1 connected\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode4 127.0.0.1:30004@40004 slave kvrockskvrockskvrockskvrockskvrocksnode3 1676818671568 1676818671569 1 connected"
- parsedCluster, err := ParseCluster(clusterStr)
- require.NoError(t, err)
- require.Equal(t, int64(1), parsedCluster.Version)
-
- require.Equal(t, 3, len(parsedCluster.Shards))
-
- require.Equal(t, []SlotRange{{Start: 0, Stop: 5460}}, parsedCluster.Shards[0].SlotRanges)
- require.Equal(t, "kvrockskvrockskvrockskvrockskvrocksnode1", parsedCluster.Shards[0].Nodes[0].ID)
- require.Equal(t, 2, len(parsedCluster.Shards[0].Nodes))
- require.Equal(t, RoleMaster, parsedCluster.Shards[0].Nodes[0].Role)
- require.Equal(t, RoleSlave, parsedCluster.Shards[0].Nodes[1].Role)
-
- require.Equal(t, []SlotRange{{Start: 5461, Stop: 10992}}, parsedCluster.Shards[1].SlotRanges)
- require.Equal(t, "kvrockskvrockskvrockskvrockskvrocksnode3", parsedCluster.Shards[1].Nodes[0].ID)
- require.Equal(t, 2, len(parsedCluster.Shards[1].Nodes))
- require.Equal(t, RoleMaster, parsedCluster.Shards[1].Nodes[0].Role)
- require.Equal(t, RoleSlave, parsedCluster.Shards[1].Nodes[1].Role)
-
- require.Equal(t, []SlotRange{{Start: 10993, Stop: 16383}}, parsedCluster.Shards[2].SlotRanges)
- require.Equal(t, "kvrockskvrockskvrockskvrockskvrocksnode5", parsedCluster.Shards[2].Nodes[0].ID)
- require.Equal(t, 2, len(parsedCluster.Shards[2].Nodes))
- require.Equal(t, RoleMaster, parsedCluster.Shards[2].Nodes[0].Role)
- require.Equal(t, RoleSlave, parsedCluster.Shards[2].Nodes[1].Role)
-
- expectedSlotString := "kvrockskvrockskvrockskvrockskvrocksnode1 127.0.0.1 30001 master - 0-5460\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode2 127.0.0.1 30002 slave kvrockskvrockskvrockskvrockskvrocksnode1\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode3 127.0.0.1 30003 master - 5461-10992\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode4 127.0.0.1 30004 slave kvrockskvrockskvrockskvrockskvrocksnode3\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode5 127.0.0.1 30005 master - 10993-16383\n" +
- "kvrockskvrockskvrockskvrockskvrocksnode6 127.0.0.1 30006 slave kvrockskvrockskvrockskvrockskvrocksnode5\n"
- gotSlotString, err := parsedCluster.ToSlotString()
- require.NoError(t, err)
- require.Equal(t, expectedSlotString, gotSlotString)
-}
diff --git a/metadata/node.go b/metadata/node.go
deleted file mode 100644
index 341e1cf..0000000
--- a/metadata/node.go
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package metadata
-
-import (
- "errors"
-
- "github.com/go-playground/validator/v10"
-)
-
-const (
- RoleMaster = "master"
- RoleSlave = "slave"
-)
-
-var (
- NodeIdLen = 40
-)
-
-var _validator = validator.New()
-
-type NodeInfo struct {
- ID string `json:"id" validate:"required"`
- Addr string `json:"addr" validate:"required"`
- Role string `json:"role" validate:"required"`
- Password string `json:"password"`
- CreatedAt int64 `json:"created_at"`
-}
-
-func (nodeInfo *NodeInfo) Validate() error {
- if len(nodeInfo.ID) == 0 {
- return errors.New("node id shouldn't be empty")
- }
- if len(nodeInfo.ID) != NodeIdLen {
- return errors.New("the length of node id must be 40")
- }
- if nodeInfo.Role != RoleMaster && nodeInfo.Role != RoleSlave {
- return errors.New("node role should be 'master' or 'slave'")
- }
- // TODO: check the node address format
- return _validator.Struct(nodeInfo)
-}
-
-func (nodeInfo *NodeInfo) IsMaster() bool {
- return nodeInfo.Role == RoleMaster
-}
diff --git a/metadata/node_test.go b/metadata/node_test.go
deleted file mode 100644
index 6778c51..0000000
--- a/metadata/node_test.go
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package metadata
-
-import (
- "strings"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestNodeInfo_Validate(t *testing.T) {
- node := &NodeInfo{}
- require.EqualError(t, node.Validate(), "node id shouldn't be empty")
- node.ID = "1234"
- require.EqualError(t, node.Validate(), "the length of node id must be 40")
- node.ID = strings.Repeat("1", NodeIdLen)
- require.EqualError(t, node.Validate(), "node role should be 'master' or 'slave'")
- node.Role = RoleMaster
- node.Addr = "1.2.3.4"
- require.NoError(t, node.Validate())
-}
diff --git a/metadata/shard.go b/metadata/shard.go
deleted file mode 100644
index 2f7ff8c..0000000
--- a/metadata/shard.go
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package metadata
-
-import (
- "errors"
- "strings"
-)
-
-type Shard struct {
- Nodes []NodeInfo `json:"nodes"`
- SlotRanges []SlotRange `json:"slot_ranges"`
- ImportSlot int `json:"import_slot"`
- MigratingSlot int `json:"migrating_slot"`
-}
-
-type Shards []Shard
-
-func (s Shards) Len() int {
- return len(s)
-}
-func (s Shards) Swap(i, j int) {
- s[i], s[j] = s[j], s[i]
-}
-func (s Shards) Less(i, j int) bool {
- return s[i].SlotRanges[0].Start < s[j].SlotRanges[0].Start
-}
-
-func NewShard() *Shard {
- return &Shard{
- Nodes: make([]NodeInfo, 0),
- SlotRanges: make([]SlotRange, 0),
- ImportSlot: -1,
- MigratingSlot: -1,
- }
-}
-
-func (shard *Shard) HasOverlap(slotRange *SlotRange) bool {
- for _, shardSlotRange := range shard.SlotRanges {
- if shardSlotRange.HasOverlap(slotRange) {
- return true
- }
- }
- return false
-}
-
-func (shard *Shard) ToSlotsString() (string, error) {
- var builder strings.Builder
- masterNodeIndex := -1
- for i, node := range shard.Nodes {
- if node.Role == RoleMaster {
- masterNodeIndex = i
- break
- }
- }
- if masterNodeIndex == -1 {
- return "", errors.New("missing master node")
- }
-
- for i, node := range shard.Nodes {
- builder.WriteString(node.ID)
- builder.WriteByte(' ')
- builder.WriteString(strings.Replace(node.Addr, ":", " ", 1))
- builder.WriteByte(' ')
- if i == masterNodeIndex {
- builder.WriteString(RoleMaster)
- builder.WriteByte(' ')
- builder.WriteByte('-')
- builder.WriteByte(' ')
- for j, slotRange := range shard.SlotRanges {
- builder.WriteString(slotRange.String())
- if j != len(shard.SlotRanges)-1 {
- builder.WriteByte(' ')
- }
- }
- } else {
- builder.WriteString(RoleSlave)
- builder.WriteByte(' ')
- builder.WriteString(shard.Nodes[masterNodeIndex].ID)
- }
- builder.WriteByte('\n')
- }
- return builder.String(), nil
-}
diff --git a/scripts/docker/docker-compose.yml b/scripts/docker/docker-compose.yml
index d483760..6ec655c 100644
--- a/scripts/docker/docker-compose.yml
+++ b/scripts/docker/docker-compose.yml
@@ -19,6 +19,34 @@
version: "3.7"
services:
+ kvrocks0:
+ image: "apache/kvrocks:nightly"
+ container_name: kvrocks0
+ ports:
+ - "7770:7770"
+ entrypoint:
+ - kvrocks
+ - "--bind"
+ - "0.0.0.0"
+ - "--port"
+ - "7770"
+ - "--cluster-enabled"
+ - "yes"
+
+ kvrocks1:
+ image: "apache/kvrocks:nightly"
+ container_name: kvrocks1
+ ports:
+ - "7771:7771"
+ entrypoint:
+ - kvrocks
+ - "--bind"
+ - "0.0.0.0"
+ - "--port"
+ - "7771"
+ - "--cluster-enabled"
+ - "yes"
+
etcd0:
image: "quay.io/coreos/etcd:latest"
container_name: etcd0
@@ -35,9 +63,9 @@ services:
- ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster
- ETCD_INITIAL_CLUSTER=etcd0=http://etcd0:2380
- ETCD_INITIAL_CLUSTER_STATE=new
+
zookeeper0:
image: "zookeeper:latest"
container_name: zookeeper0
ports:
- "2181:2181"
-
diff --git a/server/api/cluster.go b/server/api/cluster.go
new file mode 100644
index 0000000..1a07327
--- /dev/null
+++ b/server/api/cluster.go
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "errors"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/helper"
+ "github.com/apache/kvrocks-controller/store"
+)
+
+type MigrateSlotRequest struct {
+ Target int `json:"target" validate:"required"`
+ Slot int `json:"slot" validate:"required"`
+ SlotOnly bool `json:"slot_only"`
+}
+
+type CreateClusterRequest struct {
+ Name string `json:"name"`
+ Nodes []string `json:"nodes"`
+ Password string `json:"password"`
+ Replicas int `json:"replicas"`
+}
+
+type ClusterHandler struct {
+ s store.Store
+}
+
+func (handler *ClusterHandler) List(c *gin.Context) {
+ namespace := c.Param("namespace")
+ clusters, err := handler.s.ListCluster(c, namespace)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseOK(c, gin.H{"clusters": clusters})
+}
+
+func (handler *ClusterHandler) Get(c *gin.Context) {
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ helper.ResponseOK(c, gin.H{"cluster": cluster})
+}
+
+func (handler *ClusterHandler) Create(c *gin.Context) {
+ namespace := c.Param("namespace")
+ var req CreateClusterRequest
+ if err := c.BindJSON(&req); err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+
+ cluster, err := store.NewCluster(req.Name, req.Nodes, req.Replicas)
+ if err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ cluster.SetPassword(req.Password)
+ if err := handler.s.CreateCluster(c, namespace, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseCreated(c, gin.H{"cluster": cluster})
+}
+
+func (handler *ClusterHandler) Remove(c *gin.Context) {
+ namespace := c.Param("namespace")
+ cluster := c.Param("cluster")
+ err := handler.s.RemoveCluster(c, namespace, cluster)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseNoContent(c)
+}
+
+func (handler *ClusterHandler) MigrateSlot(c *gin.Context) {
+ namespace := c.Param("namespace")
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+
+ var req MigrateSlotRequest
+ if err := c.BindJSON(&req); err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+
+ err := cluster.MigrateSlot(c, req.Slot, req.Target, req.SlotOnly)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+
+ if req.SlotOnly {
+ err = handler.s.UpdateCluster(c, namespace, cluster)
+ } else {
+ // The version should be increased after the slot migration is done
+ err = handler.s.SetCluster(c, namespace, cluster)
+ }
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseOK(c, gin.H{"cluster": cluster})
+}
+
+func (handler *ClusterHandler) Import(c *gin.Context) {
+ namespace := c.Param("namespace")
+ clusterName := c.Param("cluster")
+ var req struct {
+ Nodes []string `json:"nodes"`
+ Password string `json:"password"`
+ }
+ if err := c.BindJSON(&req); err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ if len(req.Nodes) == 0 {
+ helper.ResponseBadRequest(c, errors.New("nodes should NOT be empty"))
+ return
+ }
+
+ firstNode := store.NewClusterNode(req.Nodes[0], req.Password)
+ clusterNodesStr, err := firstNode.GetClusterNodesString(c)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ cluster, err := store.ParseCluster(clusterNodesStr)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ cluster.SetPassword(req.Password)
+
+ cluster.Name = clusterName
+ if err := handler.s.CreateCluster(c, namespace, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseOK(c, gin.H{"cluster": cluster})
+}
diff --git a/server/api/cluster_test.go b/server/api/cluster_test.go
new file mode 100644
index 0000000..784128b
--- /dev/null
+++ b/server/api/cluster_test.go
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/middleware"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+func TestClusterBasics(t *testing.T) {
+ ns := "test-ns"
+ handler := &ClusterHandler{s: store.NewClusterStore(engine.NewMock())}
+
+ runCreate := func(t *testing.T, name string, expectedStatusCode int) {
+ testCreateRequest := &CreateClusterRequest{
+ Name: name,
+ Nodes: []string{"127.0.0.1:1234", "127.0.0.1:1235", "127.0.0.1:1236", "127.0.0.1:1237"},
+ Replicas: 2,
+ }
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ body, err := json.Marshal(testCreateRequest)
+ require.NoError(t, err)
+
+ ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}}
+
+ handler.Create(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runGet := func(t *testing.T, name string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: name}}
+
+ middleware.RequiredCluster(ctx)
+ if recorder.Code != http.StatusOK {
+ return
+ }
+ handler.Get(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runRemove := func(t *testing.T, name string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: name}}
+
+ middleware.RequiredCluster(ctx)
+ if recorder.Code != http.StatusOK {
+ return
+ }
+ handler.Remove(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ t.Run("create cluster", func(t *testing.T) {
+ runCreate(t, "test-cluster", http.StatusCreated)
+ runCreate(t, "test-cluster", http.StatusConflict)
+ })
+
+ t.Run("get cluster", func(t *testing.T) {
+ runGet(t, "test-cluster", http.StatusOK)
+ runGet(t, "not-exist", http.StatusNotFound)
+ })
+
+ t.Run("list cluster", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}}
+
+ handler.List(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data struct {
+ Clusters []string `json:"clusters"`
+ } `json:"data"`
+ }
+ require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &rsp))
+ require.ElementsMatch(t, []string{"test-cluster"}, rsp.Data.Clusters)
+ })
+
+ t.Run("migrate slot only", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: "test-cluster"}}
+ testMigrateReq := &MigrateSlotRequest{
+ Slot: 3,
+ SlotOnly: true,
+ Target: 1,
+ }
+ body, err := json.Marshal(testMigrateReq)
+ require.NoError(t, err)
+ ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+
+ before, err := handler.s.GetCluster(ctx, ns, "test-cluster")
+ require.NoError(t, err)
+ require.EqualValues(t, store.SlotRange{Start: 0, Stop: 8191}, before.Shards[0].SlotRanges[0])
+ require.EqualValues(t, store.SlotRange{Start: 8192, Stop: store.MaxSlotID}, before.Shards[1].SlotRanges[0])
+
+ middleware.RequiredCluster(ctx)
+ if recorder.Code != http.StatusOK {
+ return
+ }
+ handler.MigrateSlot(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ after, err := handler.s.GetCluster(ctx, ns, "test-cluster")
+ require.NoError(t, err)
+
+ require.EqualValues(t, before.Version.Inc(), after.Version.Load())
+ require.Len(t, after.Shards[0].SlotRanges, 2)
+ require.EqualValues(t, store.SlotRange{Start: 0, Stop: 2}, after.Shards[0].SlotRanges[0])
+ require.EqualValues(t, store.SlotRange{Start: 4, Stop: 8191}, after.Shards[0].SlotRanges[1])
+ require.Len(t, after.Shards[1].SlotRanges, 2)
+ require.EqualValues(t, store.SlotRange{Start: 3, Stop: 3}, after.Shards[1].SlotRanges[0])
+ require.EqualValues(t, store.SlotRange{Start: 8192, Stop: store.MaxSlotID}, after.Shards[1].SlotRanges[1])
+ })
+
+ t.Run("remove cluster", func(t *testing.T) {
+ runRemove(t, "test-cluster", http.StatusNoContent)
+ runRemove(t, "not-exist", http.StatusNotFound)
+ })
+
+}
+
+func TestClusterImport(t *testing.T) {
+ ns := "test-ns"
+ clusterName := "test-cluster-import"
+ handler := &ClusterHandler{s: store.NewClusterStore(engine.NewMock())}
+ // cluster import must be done on a real cluster
+ testNodeAddr := "127.0.0.1:7770"
+ clusterNode := store.NewClusterNode(testNodeAddr, "")
+ cluster, err := store.NewCluster(clusterName, []string{testNodeAddr}, 1)
+ require.NoError(t, err)
+ ctx := context.Background()
+ require.NoError(t, cluster.Reset(ctx))
+ require.NoError(t, clusterNode.SyncClusterInfo(ctx, cluster))
+ defer func() {
+ // clean up the cluster information to avoid affecting other tests
+ require.NoError(t, cluster.Reset(ctx))
+ }()
+
+ var req struct {
+ Nodes []string `json:"nodes"`
+ }
+ req.Nodes = []string{testNodeAddr}
+
+ recorder := httptest.NewRecorder()
+ testCtx := GetTestContext(recorder)
+ body, err := json.Marshal(req)
+ require.NoError(t, err)
+
+ testCtx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+ testCtx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: "test-cluster-import"}}
+ handler.Import(testCtx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data struct {
+ Cluster *store.Cluster `json:"cluster"`
+ } `json:"data"`
+ }
+ require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &rsp))
+ require.Len(t, rsp.Data.Cluster.Shards, 1)
+ require.Len(t, rsp.Data.Cluster.Shards[0].Nodes, 1)
+ require.Equal(t, testNodeAddr, rsp.Data.Cluster.Shards[0].Nodes[0].Addr())
+}
+
+// TestClusterMigrateData only test if the API works well here
+// and for the migration status will be tested in the controller
+func TestClusterMigrateData(t *testing.T) {
+ ns := "test-ns"
+ clusterName := "test-cluster"
+ handler := &ClusterHandler{s: store.NewClusterStore(engine.NewMock())}
+ cluster, err := store.NewCluster(clusterName, []string{"127.0.0.1:7770", "127.0.0.1:7771"}, 1)
+ require.NoError(t, err)
+ ctx := context.Background()
+ require.NoError(t, cluster.Reset(ctx))
+ defer func() {
+ require.NoError(t, cluster.Reset(ctx))
+ }()
+ for _, shard := range cluster.Shards {
+ for _, node := range shard.Nodes {
+ require.NoError(t, node.SyncClusterInfo(ctx, cluster))
+ }
+ }
+
+ recorder := httptest.NewRecorder()
+ reqCtx := GetTestContext(recorder)
+ reqCtx.Set(consts.ContextKeyStore, handler.s)
+ reqCtx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: "test-cluster"}}
+ testMigrateReq := &MigrateSlotRequest{
+ Slot: 0,
+ Target: 1,
+ }
+ body, err := json.Marshal(testMigrateReq)
+ require.NoError(t, err)
+ reqCtx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+ middleware.RequiredCluster(reqCtx)
+ if recorder.Code != http.StatusOK {
+ return
+ }
+ handler.MigrateSlot(reqCtx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ gotCluster, err := handler.s.GetCluster(ctx, ns, "test-cluster")
+ require.NoError(t, err)
+ require.EqualValues(t, 1, gotCluster.Version.Load())
+ require.Len(t, gotCluster.Shards[0].SlotRanges, 1)
+ require.EqualValues(t, 0, gotCluster.Shards[0].MigratingSlot)
+ require.EqualValues(t, 1, gotCluster.Shards[0].TargetShardIndex)
+}
diff --git a/cmd/client/executor_test.go b/server/api/handler.go
similarity index 66%
rename from cmd/client/executor_test.go
rename to server/api/handler.go
index 75232ea..262b4a2 100644
--- a/cmd/client/executor_test.go
+++ b/server/api/handler.go
@@ -17,20 +17,23 @@
* under the License.
*
*/
-package main
-import (
- "testing"
+package api
- "github.com/stretchr/testify/require"
-)
+import "github.com/apache/kvrocks-controller/store"
-func Test_create(t *testing.T) {
- request := NewRequest(config.Endpoint)
- promptCtx := NewPromptContext()
- executor := NewExecutor(promptCtx, request)
+type Handler struct {
+ Namespace *NamespaceHandler
+ Cluster *ClusterHandler
+ Shard *ShardHandler
+ Node *NodeHandler
+}
- executor.promptCtx.state = promptStateNamespace
- words := [...]string{"create", "cluster"}
- require.EqualError(t, executor.create(words[:]), ErrWrongArguments.Error())
+func NewHandler(s *store.ClusterStore) *Handler {
+ return &Handler{
+ Namespace: &NamespaceHandler{s: s},
+ Cluster: &ClusterHandler{s: s},
+ Shard: &ShardHandler{s: s},
+ Node: &NodeHandler{s: s},
+ }
}
diff --git a/server/namespace.go b/server/api/namespace.go
similarity index 61%
rename from server/namespace.go
rename to server/api/namespace.go
index 42a172b..505b979 100644
--- a/server/namespace.go
+++ b/server/api/namespace.go
@@ -17,36 +17,45 @@
* under the License.
*
*/
-package server
+package api
import (
"errors"
- "github.com/apache/kvrocks-controller/storage"
+ "github.com/apache/kvrocks-controller/consts"
+
+ "github.com/apache/kvrocks-controller/server/helper"
+
"github.com/gin-gonic/gin"
+
+ "github.com/apache/kvrocks-controller/store"
)
type NamespaceHandler struct {
- storage *storage.Storage
+ s store.Store
}
func (handler *NamespaceHandler) List(c *gin.Context) {
- namespaces, err := handler.storage.ListNamespace(c)
+ namespaces, err := handler.s.ListNamespace(c)
if err != nil {
- responseError(c, err)
+ helper.ResponseError(c, err)
return
}
- responseOK(c, gin.H{"namespaces": namespaces})
+ helper.ResponseOK(c, gin.H{"namespaces": namespaces})
}
func (handler *NamespaceHandler) Exists(c *gin.Context) {
namespace := c.Param("namespace")
- ok, err := handler.storage.IsNamespaceExists(c, namespace)
+ ok, err := handler.s.ExistsNamespace(c, namespace)
if err != nil {
- responseError(c, err)
+ helper.ResponseError(c, err)
return
}
- responseOK(c, gin.H{"exists": ok})
+ if !ok {
+ helper.ResponseError(c, consts.ErrNotFound)
+ return
+ }
+ helper.ResponseOK(c, nil)
}
func (handler *NamespaceHandler) Create(c *gin.Context) {
@@ -54,26 +63,27 @@ func (handler *NamespaceHandler) Create(c *gin.Context) {
Namespace string `json:"namespace"`
}
if err := c.BindJSON(&request); err != nil {
- responseBadRequest(c, err)
+ helper.ResponseBadRequest(c, err)
return
}
+
if len(request.Namespace) == 0 {
- responseBadRequest(c, errors.New("namespace should NOT be empty"))
+ helper.ResponseBadRequest(c, errors.New("namespace should NOT be empty"))
return
}
- if err := handler.storage.CreateNamespace(c, request.Namespace); err != nil {
- responseError(c, err)
+ if err := handler.s.CreateNamespace(c, request.Namespace); err != nil {
+ helper.ResponseError(c, err)
return
}
- responseCreated(c, "created")
+ helper.ResponseCreated(c, gin.H{"namespace": request.Namespace})
}
func (handler *NamespaceHandler) Remove(c *gin.Context) {
namespace := c.Param("namespace")
- if err := handler.storage.RemoveNamespace(c, namespace); err != nil {
- responseError(c, err)
+ if err := handler.s.RemoveNamespace(c, namespace); err != nil {
+ helper.ResponseError(c, err)
return
}
- responseOK(c, "ok")
+ helper.ResponseNoContent(c)
}
diff --git a/server/api/namespace_test.go b/server/api/namespace_test.go
new file mode 100644
index 0000000..5a09ea0
--- /dev/null
+++ b/server/api/namespace_test.go
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "testing"
+
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/gin-gonic/gin"
+)
+
+func GetTestContext(recorder *httptest.ResponseRecorder) *gin.Context {
+ gin.SetMode(gin.TestMode)
+ ctx, _ := gin.CreateTestContext(recorder)
+ ctx.Request = &http.Request{
+ Header: make(http.Header),
+ URL: &url.URL{},
+ }
+ return ctx
+}
+
+func TestNamespaceBasics(t *testing.T) {
+ handler := &NamespaceHandler{s: store.NewClusterStore(engine.NewMock())}
+
+ runCreate := func(t *testing.T, ns string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Request.Body = io.NopCloser(bytes.NewBufferString(fmt.Sprintf("{\"namespace\":\"%s\"}", ns)))
+ handler.Create(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runExists := func(t *testing.T, ns string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}}
+ handler.Exists(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runRemove := func(t *testing.T, ns string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}}
+ handler.Remove(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ t.Run("create namespace", func(t *testing.T) {
+ runCreate(t, "test0", http.StatusCreated)
+ runCreate(t, "test1", http.StatusCreated)
+ runCreate(t, "test0", http.StatusConflict)
+ })
+
+ t.Run("exits namespace", func(t *testing.T) {
+ runExists(t, "test0", http.StatusOK)
+ runExists(t, "not-exists", http.StatusNotFound)
+ })
+
+ t.Run("list namespace", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ handler.List(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data map[string][]string `json:"data"`
+ }
+ require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &rsp))
+ require.ElementsMatch(t, []string{"test0", "test1"}, rsp.Data["namespaces"])
+ })
+
+ t.Run("remove namespace", func(t *testing.T) {
+ for _, ns := range []string{"test0", "test1"} {
+ runRemove(t, ns, http.StatusNoContent)
+ runRemove(t, ns, http.StatusNotFound)
+ }
+ })
+}
diff --git a/server/api/node.go b/server/api/node.go
new file mode 100644
index 0000000..47d89e3
--- /dev/null
+++ b/server/api/node.go
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "strconv"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/helper"
+ "github.com/gin-gonic/gin"
+
+ "github.com/apache/kvrocks-controller/store"
+)
+
+type NodeHandler struct {
+ s store.Store
+}
+
+func (handler *NodeHandler) List(c *gin.Context) {
+ shard, _ := c.MustGet(consts.ContextKeyClusterShard).(*store.Shard)
+ helper.ResponseOK(c, gin.H{"nodes": shard.Nodes})
+}
+
+func (handler *NodeHandler) Create(c *gin.Context) {
+ ns := c.Param("namespace")
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ var req struct {
+ Addr string `json:"addr" binding:"required"`
+ Role string `json:"role" binding:"required"`
+ Password string `json:"password"`
+ }
+ if err := c.ShouldBindJSON(&req); err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ shardIndex, _ := strconv.Atoi(c.Param("shard"))
+ err := cluster.AddNode(shardIndex, req.Addr, req.Role, req.Password)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ if err := handler.s.UpdateCluster(c, ns, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseCreated(c, nil)
+}
+
+func (handler *NodeHandler) Remove(c *gin.Context) {
+ ns := c.Param("namespace")
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ shardIndex, _ := strconv.Atoi(c.Param("shard"))
+ err := cluster.RemoveNode(shardIndex, c.Param("id"))
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ if err := handler.s.UpdateCluster(c, ns, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseNoContent(c)
+}
diff --git a/server/api/node_test.go b/server/api/node_test.go
new file mode 100644
index 0000000..d216f81
--- /dev/null
+++ b/server/api/node_test.go
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/middleware"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNodeBasics(t *testing.T) {
+ ns := "test-ns"
+ cluster, err := store.NewCluster("test-cluster", []string{"127.0.0.1:1234", "127.0.0.1:1235"}, 2)
+ require.NoError(t, err)
+
+ handler := &NodeHandler{s: store.NewClusterStore(engine.NewMock())}
+ require.NoError(t, handler.s.CreateCluster(context.Background(), ns, cluster))
+
+ runCreate := func(t *testing.T, addr, role string, expectedStatusCode int) {
+ var req struct {
+ Addr string `json:"addr"`
+ Role string `json:"role"`
+ }
+ req.Addr = addr
+ req.Role = role
+
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ body, err := json.Marshal(req)
+ require.NoError(t, err)
+
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+ ctx.Params = []gin.Param{
+ {Key: "namespace", Value: ns},
+ {Key: "cluster", Value: cluster.Name},
+ {Key: "shard", Value: "0"}}
+
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Create(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runRemove := func(t *testing.T, nodeID string, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns},
+ {Key: "cluster", Value: cluster.Name},
+ {Key: "shard", Value: "0"},
+ {Key: "id", Value: nodeID}}
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Remove(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ t.Run("create node", func(t *testing.T) {
+ runCreate(t, "127.0.0.1:1236", "slave", http.StatusCreated)
+ runCreate(t, "127.0.0.1:1236", "master", http.StatusConflict)
+ runCreate(t, "127.0.0.1:1237", "master", http.StatusConflict)
+ })
+
+ t.Run("list node", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns},
+ {Key: "cluster", Value: cluster.Name},
+ {Key: "shard", Value: "0"}}
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.List(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data struct {
+ Nodes []*store.ClusterNode `json:"nodes"`
+ } `json:"data"`
+ }
+ err := json.Unmarshal(recorder.Body.Bytes(), &rsp)
+ require.NoError(t, err)
+ require.Len(t, rsp.Data.Nodes, 3)
+ })
+
+ t.Run("remove node", func(t *testing.T) {
+ runRemove(t, cluster.Shards[0].Nodes[0].ID(), http.StatusBadRequest)
+ runRemove(t, cluster.Shards[0].Nodes[1].ID(), http.StatusNoContent)
+ })
+}
diff --git a/server/api/shard.go b/server/api/shard.go
new file mode 100644
index 0000000..9b36ced
--- /dev/null
+++ b/server/api/shard.go
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "errors"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/helper"
+ "github.com/apache/kvrocks-controller/store"
+)
+
+type ShardHandler struct {
+ s store.Store
+}
+
+type SlotsRequest struct {
+ Slots []string `json:"slots" validate:"required"`
+}
+
+type CreateShardRequest struct {
+ Master *store.ClusterNode `json:"master"`
+ Slaves []store.ClusterNode `json:"slaves"`
+}
+
+func (handler *ShardHandler) List(c *gin.Context) {
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ helper.ResponseOK(c, gin.H{"shards": cluster.Shards})
+}
+
+func (handler *ShardHandler) Get(c *gin.Context) {
+ shard, _ := c.MustGet(consts.ContextKeyClusterShard).(*store.Shard)
+ helper.ResponseOK(c, gin.H{"shard": shard})
+}
+
+func (handler *ShardHandler) Create(c *gin.Context) {
+ ns := c.Param("namespace")
+ var req struct {
+ Nodes []string `json:"nodes"`
+ Password string `json:"password"`
+ }
+ if err := c.BindJSON(&req); err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ if len(req.Nodes) == 0 {
+ helper.ResponseBadRequest(c, errors.New("nodes should NOT be empty"))
+ return
+ }
+ nodes := make([]store.Node, 0, len(req.Nodes))
+ for i, addr := range req.Nodes {
+ node := store.NewClusterNode(addr, req.Password)
+ if i == 0 {
+ node.SetRole(store.RoleMaster)
+ } else {
+ node.SetRole(store.RoleSlave)
+ }
+ nodes = append(nodes, node)
+ }
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ newShard := store.NewShard()
+ newShard.Nodes = nodes
+ cluster.Shards = append(cluster.Shards, newShard)
+ if err := handler.s.UpdateCluster(c, ns, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseCreated(c, gin.H{"shard": newShard})
+}
+
+func (handler *ShardHandler) Remove(c *gin.Context) {
+ ns := c.Param("namespace")
+ shardIdx, err := strconv.Atoi(c.Param("shard"))
+ if err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+
+ if shardIdx < 0 || shardIdx >= len(cluster.Shards) {
+ helper.ResponseBadRequest(c, consts.ErrIndexOutOfRange)
+ return
+ }
+ if cluster.Shards[shardIdx].IsServicing() {
+ helper.ResponseBadRequest(c, consts.ErrShardIsServicing)
+ return
+ }
+ cluster.Shards = append(cluster.Shards[:shardIdx], cluster.Shards[shardIdx+1:]...)
+ if err := handler.s.UpdateCluster(c, ns, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseNoContent(c)
+}
+
+func (handler *ShardHandler) Failover(c *gin.Context) {
+ ns := c.Param("namespace")
+ cluster, _ := c.MustGet(consts.ContextKeyCluster).(*store.Cluster)
+ // We have checked this if statement in middleware.RequiredClusterShard
+ shardIndex, _ := strconv.Atoi(c.Param("shard"))
+ newMasterNodeID, err := cluster.PromoteNewMaster(c, shardIndex, "", "")
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ if err := handler.s.UpdateCluster(c, ns, cluster); err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+ helper.ResponseOK(c, gin.H{"new_master_id": newMasterNodeID})
+}
diff --git a/server/api/shard_test.go b/server/api/shard_test.go
new file mode 100644
index 0000000..1bfaa53
--- /dev/null
+++ b/server/api/shard_test.go
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package api
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/atomic"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/middleware"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+func TestShardBasics(t *testing.T) {
+ ns := "test-ns"
+ clusterName := "test-cluster"
+ handler := &ShardHandler{s: store.NewClusterStore(engine.NewMock())}
+
+ // create a test cluster
+ shard := store.NewShard()
+ shard.SlotRanges = []store.SlotRange{{Start: 0, Stop: 16383}}
+ shard.Nodes = []store.Node{store.NewClusterNode("127.0.0.1:1234", "")}
+ err := handler.s.CreateCluster(context.Background(), ns, &store.Cluster{
+ Name: clusterName,
+ Version: *atomic.NewInt64(1),
+ Shards: []*store.Shard{shard},
+ })
+ require.NoError(t, err)
+
+ runCreate := func(t *testing.T, expectedStatusCode int) {
+ var req struct {
+ Nodes []string `json:"nodes"`
+ }
+ req.Nodes = []string{"127.0.0.1:1235", "127.0.0.1:1236"}
+
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: clusterName}}
+ body, err := json.Marshal(req)
+ require.NoError(t, err)
+ ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+
+ middleware.RequiredCluster(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Create(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ runRemove := func(t *testing.T, shardIndex, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{
+ {Key: "namespace", Value: ns},
+ {Key: "cluster", Value: clusterName},
+ {Key: "shard", Value: strconv.Itoa(shardIndex)}}
+
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Remove(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ t.Run("create shard", func(t *testing.T) {
+ runCreate(t, http.StatusCreated)
+ })
+
+ t.Run("get shard", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{
+ {Key: "namespace", Value: ns},
+ {Key: "cluster", Value: clusterName},
+ {Key: "shard", Value: "1"}}
+
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Get(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data struct {
+ Shard *store.Shard `json:"shard"`
+ } `json:"data"`
+ }
+ err := json.Unmarshal(recorder.Body.Bytes(), &rsp)
+ require.NoError(t, err)
+ require.Len(t, rsp.Data.Shard.Nodes, 2)
+
+ var nodeAddrs []string
+ for _, node := range rsp.Data.Shard.Nodes {
+ nodeAddrs = append(nodeAddrs, node.Addr())
+ }
+ require.ElementsMatch(t, []string{"127.0.0.1:1235", "127.0.0.1:1236"}, nodeAddrs)
+ require.EqualValues(t, -1, rsp.Data.Shard.MigratingSlot)
+ require.EqualValues(t, -1, rsp.Data.Shard.TargetShardIndex)
+ })
+
+ t.Run("list shards", func(t *testing.T) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{{Key: "namespace", Value: ns}, {Key: "cluster", Value: clusterName}}
+
+ middleware.RequiredCluster(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.List(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+
+ var rsp struct {
+ Data struct {
+ Shards []*store.Shard `json:"shards"`
+ } `json:"data"`
+ }
+ err := json.Unmarshal(recorder.Body.Bytes(), &rsp)
+ require.NoError(t, err)
+ require.Len(t, rsp.Data.Shards, 2)
+ })
+
+ t.Run("remove shard", func(t *testing.T) {
+ // shard 0 is servicing
+ runRemove(t, 0, http.StatusBadRequest)
+ runRemove(t, 1, http.StatusNoContent)
+ })
+}
+
+func TestClusterFailover(t *testing.T) {
+ ns := "test-ns"
+ clusterName := "test-cluster-failover"
+ handler := &ShardHandler{s: store.NewClusterStore(engine.NewMock())}
+ cluster, err := store.NewCluster(clusterName, []string{"127.0.0.1:7770", "127.0.0.1:7771"}, 2)
+ require.NoError(t, err)
+ node0, _ := cluster.Shards[0].Nodes[0].(*store.ClusterNode)
+ node1, _ := cluster.Shards[0].Nodes[1].(*store.ClusterNode)
+
+ runFailover := func(t *testing.T, shardIndex, expectedStatusCode int) {
+ recorder := httptest.NewRecorder()
+ ctx := GetTestContext(recorder)
+ ctx.Set(consts.ContextKeyStore, handler.s)
+ ctx.Params = []gin.Param{
+ {Key: "namespace", Value: ns},
+ {Key: "cluster", Value: clusterName},
+ {Key: "shard", Value: strconv.Itoa(shardIndex)}}
+
+ middleware.RequiredClusterShard(ctx)
+ require.Equal(t, http.StatusOK, recorder.Code)
+ handler.Failover(ctx)
+ require.Equal(t, expectedStatusCode, recorder.Code)
+ }
+
+ t.Run("failover is good", func(t *testing.T) {
+ ctx := context.Background()
+ masterCli := node0.GetClient()
+ replicaCli := node1.GetClient()
+ require.NoError(t, masterCli.ClusterResetHard(ctx).Err())
+ require.NoError(t, replicaCli.ClusterResetHard(ctx).Err())
+ defer func() {
+ require.NoError(t, masterCli.ClusterResetHard(ctx).Err())
+ require.NoError(t, replicaCli.ClusterResetHard(ctx).Err())
+ }()
+
+ require.NoError(t, handler.s.CreateCluster(ctx, ns, cluster))
+ runFailover(t, 0, http.StatusOK)
+ })
+
+ t.Run("cluster topology is good", func(t *testing.T) {
+ ctx := context.Background()
+ gotCluster, err := handler.s.GetCluster(ctx, ns, clusterName)
+ require.NoError(t, err)
+ require.EqualValues(t, 2, gotCluster.Version.Load())
+ require.Len(t, gotCluster.Shards, 1)
+ for _, node := range gotCluster.Shards[0].Nodes {
+ if node.ID() == node0.ID() {
+ // become slave now
+ require.False(t, node.IsMaster())
+ } else {
+ require.True(t, node.IsMaster())
+ }
+ }
+
+ // sync cluster info to each node
+ require.NoError(t, node0.SyncClusterInfo(ctx, gotCluster))
+ require.NoError(t, node1.SyncClusterInfo(ctx, gotCluster))
+
+ clusterNodeInfo0, err := node0.GetClusterNodeInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, "slave", clusterNodeInfo0.Role)
+ clusterNodeInfo1, err := node1.GetClusterNodeInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, "master", clusterNodeInfo1.Role)
+ })
+}
diff --git a/server/cluster.go b/server/cluster.go
deleted file mode 100644
index 92333a3..0000000
--- a/server/cluster.go
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "errors"
- "fmt"
- "time"
-
- "github.com/apache/kvrocks-controller/controller/failover"
- "github.com/apache/kvrocks-controller/util"
- "github.com/gin-gonic/gin"
-
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
-)
-
-type CreateClusterRequest struct {
- Name string `json:"name"`
- Nodes []string `json:"nodes"`
- Password string `json:"password"`
- Replicas int `json:"replicas"`
-}
-
-func (req *CreateClusterRequest) validate() error {
- if len(req.Name) == 0 {
- return fmt.Errorf("cluster name should NOT be empty")
- }
- if len(req.Nodes) == 0 {
- return errors.New("cluster nodes should NOT be empty")
- }
- if !util.IsUniqueSlice(req.Nodes) {
- return errors.New("cluster nodes should NOT be duplicated")
- }
-
- invalidNodes := make([]string, 0)
- for _, node := range req.Nodes {
- if !util.IsHostPort(node) {
- invalidNodes = append(invalidNodes, node)
- }
- }
- if len(invalidNodes) > 0 {
- return fmt.Errorf("invalid node addresses: %v", invalidNodes)
- }
-
- if req.Replicas == 0 {
- req.Replicas = 1
- }
- if len(req.Nodes)%req.Replicas != 0 {
- return errors.New("cluster nodes should be divisible by replica")
- }
- return nil
-}
-
-type ClusterHandler struct {
- storage *storage.Storage
-}
-
-func (handler *ClusterHandler) List(c *gin.Context) {
- namespace := c.Param("namespace")
- clusters, err := handler.storage.ListCluster(c, namespace)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, gin.H{"clusters": clusters})
-}
-
-func (handler *ClusterHandler) Get(c *gin.Context) {
- namespace := c.Param("namespace")
- clusterName := c.Param("cluster")
- cluster, err := handler.storage.GetClusterInfo(c, namespace, clusterName)
- if err != nil {
- responseError(c, metadata.ErrEntryNoExists)
- return
- }
- responseOK(c, gin.H{"cluster": cluster})
-}
-
-func (handler *ClusterHandler) Create(c *gin.Context) {
- namespace := c.Param("namespace")
-
- var req CreateClusterRequest
- if err := c.BindJSON(&req); err != nil {
- responseBadRequest(c, err)
- return
- }
- if err := req.validate(); err != nil {
- responseBadRequest(c, err)
- return
- }
-
- if c.GetHeader(consts.HeaderDontDetectHost) != "true" {
- for _, node := range req.Nodes {
- if err := util.DetectClusterNode(c, &metadata.NodeInfo{
- Addr: node,
- Password: req.Password,
- }); err != nil {
- responseBadRequest(c, err)
- return
- }
- }
- }
-
- replicas := req.Replicas
- shards := make([]metadata.Shard, len(req.Nodes)/replicas)
- slotRanges := metadata.SpiltSlotRange(len(shards))
- for i := range shards {
- shards[i].Nodes = make([]metadata.NodeInfo, 0)
- for j := 0; j < replicas; j++ {
- nodeAddr := req.Nodes[i*replicas+j]
- role := metadata.RoleMaster
- if j != 0 {
- role = metadata.RoleSlave
- }
- shards[i].Nodes = append(shards[i].Nodes, metadata.NodeInfo{
- ID: util.GenerateNodeID(),
- Addr: nodeAddr,
- Password: req.Password,
- Role: role,
- CreatedAt: time.Now().Unix(),
- })
- }
- shards[i].SlotRanges = append(shards[i].SlotRanges, slotRanges[i])
- shards[i].MigratingSlot = -1
- shards[i].ImportSlot = -1
- }
- err := handler.storage.CreateCluster(c, namespace, &metadata.Cluster{
- Version: 1,
- Name: req.Name,
- Shards: shards,
- })
- if err != nil {
- responseError(c, err)
- return
- }
- responseCreated(c, "created")
-}
-
-func (handler *ClusterHandler) Remove(c *gin.Context) {
- namespace := c.Param("namespace")
- cluster := c.Param("cluster")
- err := handler.storage.RemoveCluster(c, namespace, cluster)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
-
-func (handler *ClusterHandler) Import(c *gin.Context) {
- namespace := c.Param("namespace")
- clusterName := c.Param("cluster")
- var req struct {
- Nodes []string `json:"nodes"`
- Password string `json:"password"`
- }
- if err := c.BindJSON(&req); err != nil {
- responseBadRequest(c, err)
- return
- }
- if len(req.Nodes) == 0 {
- responseBadRequest(c, errors.New("nodes should NOT be empty"))
- return
- }
-
- clusterNodesStr, err := util.ClusterNodesCmd(c, &metadata.NodeInfo{
- Addr: req.Nodes[0],
- Password: req.Password,
- })
- if err != nil {
- responseError(c, err)
- return
- }
- clusterInfo, err := metadata.ParseCluster(clusterNodesStr)
- if err != nil {
- responseError(c, err)
- return
- }
- clusterInfo.SetPassword(req.Password)
-
- clusterInfo.Name = clusterName
- if err := handler.storage.CreateCluster(c, namespace, clusterInfo); err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
-
-func (handler *ClusterHandler) GetFailOverTasks(c *gin.Context) {
- namespace := c.Param("namespace")
- cluster := c.Param("cluster")
- typ := c.Param("type")
- failover, _ := c.MustGet(consts.ContextKeyFailover).(*failover.FailOver)
- tasks, err := failover.GetTasks(c, namespace, cluster, typ)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, gin.H{"tasks": tasks})
-}
diff --git a/server/cluster_test.go b/server/cluster_test.go
deleted file mode 100644
index f0b2574..0000000
--- a/server/cluster_test.go
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "fmt"
- "net/http"
- "testing"
-
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/metadata"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/require"
-)
-
-func createTestCluster(t *testing.T, ns, clusterName string) {
- clusterURI := fmt.Sprintf("%s/%s/clusters", "/api/v1/namespaces", ns)
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Post(clusterURI).
- Header(consts.HeaderDontDetectHost, "true").
- JSON(CreateClusterRequest{
- Name: clusterName,
- Nodes: []string{
- "1.1.1.1:6666",
- "1.1.1.2:6666",
- "1.1.1.3:6666",
- },
- Replicas: 1,
- }).
- Expect(t).
- Status(http.StatusCreated).
- End()
-}
-
-func TestCluster(t *testing.T) {
- ns := uuid.NewString()
- clusterURI := fmt.Sprintf("%s/%s/clusters", "/api/v1/namespaces", ns)
- clusterName := uuid.NewString()
-
- createTestNamespace(t, ns)
-
- t.Run("Create cluster", func(t *testing.T) {
- createTestCluster(t, ns, clusterName)
- })
-
- t.Run("List clusters", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Clusters []string `json:"clusters"`
- }
- }
- newRequest().
- Get(clusterURI).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Contains(t, result.Data.Clusters, clusterName)
- })
-
- t.Run("Get cluster", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Cluster metadata.Cluster `json:"cluster"`
- }
- }
- newRequest().
- Get(clusterURI + "/" + clusterName).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Equal(t, clusterName, result.Data.Cluster.Name)
- require.Equal(t, 3, len(result.Data.Cluster.Shards))
- })
-
- t.Run("Get non-exists cluster", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Cluster metadata.Cluster `json:"cluster"`
- }
- }
- newRequest().
- Get(clusterURI + "/non-exists-cluster").
- Expect(t).
- Status(http.StatusNotFound).
- End().
- JSON(&result)
- })
-
- t.Run("Delete cluster", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Delete(clusterURI + "/" + clusterName).
- Expect(t).
- Status(http.StatusOK).
- End()
- })
-}
diff --git a/server/controller.go b/server/controller.go
deleted file mode 100644
index 942f99e..0000000
--- a/server/controller.go
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/gin-gonic/gin"
-)
-
-func Leader(c *gin.Context) {
- stor := c.MustGet(consts.ContextKeyStorage).(*storage.Storage)
- responseOK(c, stor.Leader())
-}
-
-func LeaderResign(c *gin.Context) {
- stor := c.MustGet(consts.ContextKeyStorage).(*storage.Storage)
- stor.Stop()
- responseOK(c, "OK")
-}
diff --git a/server/util.go b/server/helper/helper.go
similarity index 70%
rename from server/util.go
rename to server/helper/helper.go
index 2c0866c..22851e2 100644
--- a/server/util.go
+++ b/server/helper/helper.go
@@ -17,13 +17,13 @@
* under the License.
*
*/
-package server
+package helper
import (
"errors"
"net/http"
- "github.com/apache/kvrocks-controller/metadata"
+ "github.com/apache/kvrocks-controller/consts"
"github.com/gin-gonic/gin"
)
@@ -37,15 +37,19 @@ type Response struct {
Data interface{} `json:"data"`
}
-func responseOK(c *gin.Context, data interface{}) {
+func ResponseOK(c *gin.Context, data interface{}) {
responseData(c, http.StatusOK, data)
}
-func responseCreated(c *gin.Context, data interface{}) {
+func ResponseCreated(c *gin.Context, data interface{}) {
responseData(c, http.StatusCreated, data)
}
-func responseBadRequest(c *gin.Context, err error) {
+func ResponseNoContent(c *gin.Context) {
+ c.JSON(http.StatusNoContent, nil)
+}
+
+func ResponseBadRequest(c *gin.Context, err error) {
c.JSON(http.StatusBadRequest, Response{
Error: &Error{Message: err.Error()},
})
@@ -57,14 +61,19 @@ func responseData(c *gin.Context, code int, data interface{}) {
})
}
-func responseError(c *gin.Context, err error) {
+func ResponseError(c *gin.Context, err error) {
code := http.StatusInternalServerError
- if errors.Is(err, metadata.ErrEntryNoExists) {
+ if errors.Is(err, consts.ErrNotFound) {
code = http.StatusNotFound
- } else if errors.Is(err, metadata.ErrEntryExisted) {
+ } else if errors.Is(err, consts.ErrIndexOutOfRange) {
+ code = http.StatusBadRequest
+ } else if errors.Is(err, consts.ErrAlreadyExists) {
code = http.StatusConflict
+ } else if errors.Is(err, consts.ErrInvalidArgument) {
+ code = http.StatusBadRequest
}
c.JSON(code, Response{
Error: &Error{Message: err.Error()},
})
+ c.Abort()
}
diff --git a/server/middleware.go b/server/middleware/middleware.go
similarity index 60%
rename from server/middleware.go
rename to server/middleware/middleware.go
index abed2d2..7955825 100644
--- a/server/middleware.go
+++ b/server/middleware/middleware.go
@@ -17,7 +17,7 @@
* under the License.
*
*/
-package server
+package middleware
import (
"errors"
@@ -25,11 +25,11 @@ import (
"strconv"
"time"
- "github.com/apache/kvrocks-controller/metadata"
+ "github.com/apache/kvrocks-controller/server/helper"
"github.com/apache/kvrocks-controller/consts"
"github.com/apache/kvrocks-controller/metrics"
- "github.com/apache/kvrocks-controller/storage"
+ "github.com/apache/kvrocks-controller/store"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
@@ -60,9 +60,9 @@ func CollectMetrics(c *gin.Context) {
}
func RedirectIfNotLeader(c *gin.Context) {
- storage, _ := c.MustGet(consts.ContextKeyStorage).(*storage.Storage)
+ storage, _ := c.MustGet(consts.ContextKeyStore).(*store.ClusterStore)
if storage.Leader() == "" {
- responseBadRequest(c, errors.New("no leader now, please retry later"))
+ c.JSON(http.StatusBadRequest, gin.H{"error": "no leader now, please retry later"})
c.Abort()
return
}
@@ -71,38 +71,60 @@ func RedirectIfNotLeader(c *gin.Context) {
c.Set(consts.HeaderIsRedirect, true)
c.Redirect(http.StatusTemporaryRedirect, "http://"+storage.Leader()+c.Request.RequestURI)
} else {
- responseBadRequest(c, errors.New("too many redirects"))
+ c.JSON(http.StatusBadRequest, gin.H{"error": "no leader now, please retry later"})
+ c.Abort()
}
- c.Abort()
return
}
c.Next()
}
-func requiredNamespace(c *gin.Context) {
- storage, _ := c.MustGet(consts.ContextKeyStorage).(*storage.Storage)
- ok, err := storage.IsNamespaceExists(c, c.Param("namespace"))
+func RequiredNamespace(c *gin.Context) {
+ s, _ := c.MustGet(consts.ContextKeyStore).(*store.ClusterStore)
+ ok, err := s.ExistsNamespace(c, c.Param("namespace"))
if err != nil {
- responseError(c, err)
+ helper.ResponseError(c, err)
return
}
if !ok {
- responseBadRequest(c, metadata.ErrEntryNoExists)
+ helper.ResponseBadRequest(c, errors.New("namespace not found"))
return
}
c.Next()
}
-func requiredCluster(c *gin.Context) {
- storage, _ := c.MustGet(consts.ContextKeyStorage).(*storage.Storage)
- ok, err := storage.IsClusterExists(c, c.Param("namespace"), c.Param("cluster"))
+func RequiredCluster(c *gin.Context) {
+ s, _ := c.MustGet(consts.ContextKeyStore).(*store.ClusterStore)
+ cluster, err := s.GetCluster(c, c.Param("namespace"), c.Param("cluster"))
if err != nil {
- responseError(c, err)
+ helper.ResponseError(c, err)
return
}
- if !ok {
- responseBadRequest(c, metadata.ErrEntryNoExists)
+
+ c.Set(consts.ContextKeyCluster, cluster)
+ c.Next()
+}
+
+func RequiredClusterShard(c *gin.Context) {
+ s, _ := c.MustGet(consts.ContextKeyStore).(*store.ClusterStore)
+ cluster, err := s.GetCluster(c, c.Param("namespace"), c.Param("cluster"))
+ if err != nil {
+ helper.ResponseError(c, err)
return
}
+
+ shardIndex, err := strconv.Atoi(c.Param("shard"))
+ if err != nil {
+ helper.ResponseBadRequest(c, err)
+ return
+ }
+ shard, err := cluster.GetShard(shardIndex)
+ if err != nil {
+ helper.ResponseError(c, err)
+ return
+ }
+
+ c.Set(consts.ContextKeyCluster, cluster)
+ c.Set(consts.ContextKeyClusterShard, shard)
c.Next()
}
diff --git a/server/namespace_test.go b/server/namespace_test.go
deleted file mode 100644
index c0f668e..0000000
--- a/server/namespace_test.go
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "fmt"
- "net/http"
- "testing"
-
- "github.com/apache/kvrocks-controller/config"
- "github.com/google/uuid"
- "github.com/steinfletcher/apitest"
- "github.com/stretchr/testify/require"
-)
-
-func newTestServer() (*Server, func() *apitest.APITest) {
- server, _ := NewServer(config.Default())
- server.initHandlers()
- return server, func() *apitest.APITest {
- return apitest.New().Handler(server.engine)
- }
-}
-
-func createTestNamespace(t *testing.T, ns string) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Post("/api/v1/namespaces").
- Body(fmt.Sprintf(`{"namespace": "%s"}`, ns)).
- Expect(t).
- Status(http.StatusCreated).
- End()
-}
-
-func TestNamespace(t *testing.T) {
- uri := "/api/v1/namespaces"
- ns := uuid.New().String()
-
- t.Run("Create namespace", func(t *testing.T) {
- createTestNamespace(t, ns)
- })
-
- t.Run("Create duplicate namespace", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Post(uri).
- Body(fmt.Sprintf(`{"namespace": "%s"}`, ns)).
- Expect(t).
- Status(http.StatusConflict).
- End()
- })
-
- t.Run("List namespaces", func(t *testing.T) {
- var result struct {
- Data struct {
- Namespaces []string `json:"namespaces"`
- } `json:"data"`
- }
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Get(uri).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Contains(t, result.Data.Namespaces, ns)
- })
-
- t.Run("Delete namespace", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Delete(uri + "/" + ns).
- Expect(t).
- Status(http.StatusOK).
- End()
-
- // double delete should return 404
- newRequest().
- Delete(uri + "/" + ns).
- Expect(t).
- Status(http.StatusNotFound).
- End()
- })
-}
diff --git a/server/node.go b/server/node.go
deleted file mode 100644
index e164cae..0000000
--- a/server/node.go
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "strconv"
- "time"
-
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/util"
- "github.com/gin-gonic/gin"
-)
-
-type NodeHandler struct {
- storage *storage.Storage
-}
-
-func (handler *NodeHandler) List(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
-
- nodes, err := handler.storage.ListNodes(c, ns, cluster, shard)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, gin.H{"nodes": nodes})
-}
-
-func (handler *NodeHandler) Create(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
-
- var nodeInfo metadata.NodeInfo
- if err := c.BindJSON(&nodeInfo); err != nil {
- responseBadRequest(c, err)
- return
- }
- nodeInfo.CreatedAt = time.Now().Unix()
- if nodeInfo.ID == "" {
- nodeInfo.ID = util.GenerateNodeID()
- }
- if err := nodeInfo.Validate(); err != nil {
- responseBadRequest(c, err)
- return
- }
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
- if c.GetHeader(consts.HeaderDontDetectHost) != "true" {
- if err := util.DetectClusterNode(c, &nodeInfo); err != nil {
- responseBadRequest(c, err)
- return
- }
- }
-
- err = handler.storage.CreateNode(c, ns, cluster, shard, &nodeInfo)
- switch err {
- case nil:
- responseCreated(c, "created")
- case metadata.ErrEntryExisted:
- responseBadRequest(c, err)
- default:
- responseError(c, err)
- }
-}
-
-func (handler *NodeHandler) Remove(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- id := c.Param("id")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
-
- if err := handler.storage.RemoveNode(c, ns, cluster, shard, id); err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
diff --git a/server/node_test.go b/server/node_test.go
deleted file mode 100644
index 5233e57..0000000
--- a/server/node_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "fmt"
- "net/http"
- "testing"
-
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/util"
- "github.com/google/uuid"
- "github.com/stretchr/testify/require"
-)
-
-func TestNodeHandler(t *testing.T) {
- ns := uuid.NewString()
- clusterName := uuid.NewString()
- nodeURI := fmt.Sprintf("/api/v1/namespaces/%s/clusters/%s/shards/0/nodes", ns, clusterName)
- nodeID := util.GenerateNodeID()
- createTestNamespace(t, ns)
- createTestCluster(t, ns, clusterName)
-
- t.Run("Create node", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Post(nodeURI).
- Header(consts.HeaderDontDetectHost, "true").
- JSON(metadata.NodeInfo{
- ID: nodeID,
- Addr: "1.1.1.5:1234",
- Role: metadata.RoleSlave,
- }).
- Expect(t).
- Status(http.StatusCreated).
- End()
- })
-
- t.Run("List node", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Nodes []metadata.NodeInfo `json:"nodes"`
- }
- }
- newRequest().
- Get(nodeURI).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Equal(t, 2, len(result.Data.Nodes))
- })
-
- t.Run("Delete node", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- newRequest().
- Delete(nodeURI + "/" + nodeID).
- Expect(t).
- Status(http.StatusOK).
- End()
-
- var result struct {
- Data struct {
- Nodes []metadata.NodeInfo `json:"nodes"`
- }
- }
- newRequest().
- Get(nodeURI).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Equal(t, 1, len(result.Data.Nodes))
- })
-}
diff --git a/server/route.go b/server/route.go
index c9e9a89..ba946a9 100644
--- a/server/route.go
+++ b/server/route.go
@@ -20,78 +20,59 @@
package server
import (
- "github.com/apache/kvrocks-controller/consts"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/server/api"
+ "github.com/apache/kvrocks-controller/server/middleware"
)
func (srv *Server) initHandlers() {
engine := srv.engine
- engine.Use(CollectMetrics, func(c *gin.Context) {
- c.Set(consts.ContextKeyStorage, srv.storage)
- c.Set(consts.ContextKeyMigrator, srv.controller.GetMigrate())
- c.Set(consts.ContextKeyFailover, srv.controller.GetFailOver())
+ engine.Use(middleware.CollectMetrics, func(c *gin.Context) {
+ c.Set(consts.ContextKeyStore, srv.store)
c.Next()
- }, RedirectIfNotLeader)
- namespace := &NamespaceHandler{storage: srv.storage}
- cluster := &ClusterHandler{storage: srv.storage}
- shard := &ShardHandler{storage: srv.storage}
- node := &NodeHandler{storage: srv.storage}
+ }, middleware.RedirectIfNotLeader)
+ handler := api.NewHandler(srv.store)
- apiTest := engine.Group("/api/test/")
- {
- controller := apiTest.Group("controller")
- controller.GET("/leader/resign", LeaderResign)
- }
engine.Any("/debug/pprof/*profile", PProf)
engine.GET("/metrics", gin.WrapH(promhttp.Handler()))
apiV1 := engine.Group("/api/v1/")
{
- controller := apiV1.Group("controller")
- {
- controller.GET("/leader", Leader)
- }
-
namespaces := apiV1.Group("namespaces")
{
- namespaces.GET("", namespace.List)
- namespaces.GET("/:namespace", namespace.Exists)
- namespaces.POST("", namespace.Create)
- namespaces.DELETE("/:namespace", namespace.Remove)
+ namespaces.GET("", handler.Namespace.List)
+ namespaces.GET("/:namespace", handler.Namespace.Exists)
+ namespaces.POST("", handler.Namespace.Create)
+ namespaces.DELETE("/:namespace", handler.Namespace.Remove)
}
clusters := namespaces.Group("/:namespace/clusters")
{
- clusters.Use(requiredNamespace)
- clusters.GET("", cluster.List)
- clusters.GET("/:cluster", cluster.Get)
- clusters.POST("", cluster.Create)
- clusters.POST("/:cluster/import", cluster.Import)
- clusters.DELETE("/:cluster", cluster.Remove)
- clusters.GET("/:cluster/failover/:type", cluster.GetFailOverTasks)
+ clusters.GET("", middleware.RequiredNamespace, handler.Cluster.List)
+ clusters.POST("", middleware.RequiredNamespace, handler.Cluster.Create)
+ clusters.GET("/:cluster", middleware.RequiredCluster, handler.Cluster.Get)
+ clusters.POST("/:cluster/import", handler.Cluster.Import)
+ clusters.DELETE("/:cluster", handler.Cluster.Remove)
+ clusters.POST("/:cluster/migrate", middleware.RequiredCluster, handler.Cluster.MigrateSlot)
}
shards := clusters.Group("/:cluster/shards")
{
- shards.Use(requiredCluster)
- shards.GET("", shard.List)
- shards.GET("/:shard", shard.Get)
- shards.POST("", shard.Create)
- shards.DELETE("/:shard", shard.Remove)
- shards.POST("/:shard/failover", shard.Failover)
- shards.POST("/:shard/slots", shard.UpdateSlots)
- shards.DELETE("/:shard/slots", shard.UpdateSlots)
- shards.POST("/migration/slot_data", shard.MigrateSlotData)
- shards.POST("/migration/slot_only", shard.MigrateSlotOnly)
+ shards.GET("", middleware.RequiredCluster, handler.Shard.List)
+ shards.POST("", middleware.RequiredCluster, handler.Shard.Create)
+ shards.GET("/:shard", middleware.RequiredClusterShard, handler.Shard.Get)
+ shards.DELETE("/:shard", middleware.RequiredCluster, handler.Shard.Remove)
+ shards.POST("/:shard/failover", middleware.RequiredClusterShard, handler.Shard.Failover)
}
nodes := shards.Group("/:shard/nodes")
{
- nodes.Use(requiredCluster)
- nodes.GET("", node.List)
- nodes.POST("", node.Create)
- nodes.DELETE("/:id", node.Remove)
+ nodes.GET("", middleware.RequiredClusterShard, handler.Node.List)
+ nodes.POST("", middleware.RequiredClusterShard, handler.Node.Create)
+ nodes.DELETE("/:id", middleware.RequiredClusterShard, handler.Node.Remove)
}
}
}
diff --git a/server/server.go b/server/server.go
index 336d647..d4b745c 100644
--- a/server/server.go
+++ b/server/server.go
@@ -27,38 +27,37 @@ import (
"strings"
"time"
+ "github.com/gin-gonic/gin"
+
"github.com/apache/kvrocks-controller/config"
"github.com/apache/kvrocks-controller/controller"
- "github.com/apache/kvrocks-controller/controller/probe"
"github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/apache/kvrocks-controller/storage/persistence"
- "github.com/apache/kvrocks-controller/storage/persistence/etcd"
- "github.com/apache/kvrocks-controller/storage/persistence/zookeeper"
- "github.com/gin-gonic/gin"
+ "github.com/apache/kvrocks-controller/store"
+ "github.com/apache/kvrocks-controller/store/engine"
+ "github.com/apache/kvrocks-controller/store/engine/etcd"
+ "github.com/apache/kvrocks-controller/store/engine/zookeeper"
)
type Server struct {
- engine *gin.Engine
- storage *storage.Storage
- healthProbe *probe.Probe
- controller *controller.Controller
- config *config.Config
- httpServer *http.Server
+ engine *gin.Engine
+ store *store.ClusterStore
+ controller *controller.Controller
+ config *config.Config
+ httpServer *http.Server
}
func NewServer(cfg *config.Config) (*Server, error) {
- var persist persistence.Persistence
+ var persist engine.Engine
var err error
switch {
case strings.EqualFold(cfg.StorageType, "etcd"):
- logger.Get().Info("Use Etcd as storage")
+ logger.Get().Info("Use Etcd as store")
persist, err = etcd.New(cfg.Addr, cfg.Etcd)
case strings.EqualFold(cfg.StorageType, "zookeeper"):
- logger.Get().Info("Use Zookeeper as storage")
+ logger.Get().Info("Use Zookeeper as store")
persist, err = zookeeper.New(cfg.Addr, cfg.Zookeeper)
default:
- logger.Get().Info("Use Etcd as default storage")
+ logger.Get().Info("Use Etcd as default store")
persist, err = etcd.New(cfg.Addr, cfg.Etcd)
}
@@ -66,23 +65,20 @@ func NewServer(cfg *config.Config) (*Server, error) {
return nil, err
}
if persist == nil {
- return nil, fmt.Errorf("no found any storage config")
- }
- storage, err := storage.NewStorage(persist)
- if err != nil {
- return nil, err
+ return nil, fmt.Errorf("no found any store config")
}
- if ok := storage.IsReady(); !ok {
- return nil, fmt.Errorf("storage is not ready")
+ clusterStore := store.NewClusterStore(persist)
+ if ok := clusterStore.IsReady(context.Background()); !ok {
+ return nil, fmt.Errorf("the cluster store is not ready")
}
- ctrl, err := controller.New(storage, cfg)
+ ctrl, err := controller.New(clusterStore, cfg.Controller)
if err != nil {
return nil, err
}
gin.SetMode(gin.ReleaseMode)
return &Server{
- storage: storage,
+ store: clusterStore,
controller: ctrl,
config: cfg,
engine: gin.New(),
@@ -121,8 +117,8 @@ func PProf(c *gin.Context) {
}
}
-func (srv *Server) Start() error {
- if err := srv.controller.Start(); err != nil {
+func (srv *Server) Start(ctx context.Context) error {
+ if err := srv.controller.Start(ctx); err != nil {
return err
}
srv.startAPIServer()
@@ -130,7 +126,7 @@ func (srv *Server) Start() error {
}
func (srv *Server) Stop() error {
- _ = srv.controller.Stop()
+ srv.controller.Close()
gracefulCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
return srv.httpServer.Shutdown(gracefulCtx)
diff --git a/server/shard.go b/server/shard.go
deleted file mode 100644
index 76c0c9b..0000000
--- a/server/shard.go
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "errors"
- "net/http"
- "strconv"
- "time"
-
- "github.com/apache/kvrocks-controller/util"
-
- "github.com/apache/kvrocks-controller/consts"
- "github.com/apache/kvrocks-controller/controller/failover"
- "github.com/apache/kvrocks-controller/controller/migrate"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage"
- "github.com/gin-gonic/gin"
-)
-
-type ShardHandler struct {
- storage *storage.Storage
-}
-
-type SlotsRequest struct {
- Slots []string `json:"slots" validate:"required"`
-}
-
-type MigrateSlotDataRequest struct {
- Source int `json:"source" validate:"required"`
- Target int `json:"target" validate:"required"`
- Slot int `json:"slot"`
-}
-
-type MigrateSlotOnlyRequest struct {
- Source int `json:"source" validate:"required"`
- Target int `json:"target" validate:"required"`
- Slots []metadata.SlotRange `json:"slots"`
-}
-
-type CreateShardRequest struct {
- Master *metadata.NodeInfo `json:"master"`
- Slaves []metadata.NodeInfo `json:"slaves"`
-}
-
-func (handler *ShardHandler) List(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
-
- shards, err := handler.storage.ListShard(c, ns, cluster)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, gin.H{"shards": shards})
-}
-
-func (handler *ShardHandler) Get(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
-
- s, err := handler.storage.GetShard(c, ns, cluster, shard)
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, gin.H{"shard": s})
-}
-
-func (handler *ShardHandler) Create(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
-
- var req struct {
- Nodes []string `json:"nodes"`
- Password string `json:"password"`
- }
- if err := c.BindJSON(&req); err != nil {
- responseBadRequest(c, err)
- return
- }
- if len(req.Nodes) == 0 {
- responseBadRequest(c, errors.New("nodes should NOT be empty"))
- return
- }
- nodes := make([]metadata.NodeInfo, len(req.Nodes))
- now := time.Now().Unix()
- for i, nodeAddr := range req.Nodes {
- nodes[i].ID = util.GenerateNodeID()
- nodes[i].Addr = nodeAddr
- if i == 0 {
- nodes[i].Role = metadata.RoleMaster
- } else {
- nodes[i].Role = metadata.RoleSlave
- }
- nodes[i].Password = req.Password
- nodes[i].CreatedAt = now
- }
- if err := handler.storage.CreateShard(c, ns, cluster, &metadata.Shard{
- Nodes: nodes,
- ImportSlot: -1,
- MigratingSlot: -1,
- }); err != nil {
- responseError(c, err)
- return
- }
- responseCreated(c, "ok")
-}
-
-func (handler *ShardHandler) Remove(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
-
- err = handler.storage.RemoveShard(c, ns, cluster, shard)
- if err != nil {
- responseError(c, err)
- return
- }
- responseData(c, http.StatusOK, nil)
-}
-
-func (handler *ShardHandler) UpdateSlots(c *gin.Context) {
- isAdd := c.Request.Method == http.MethodPost
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
- var payload SlotsRequest
- if err := c.BindJSON(&payload); err != nil {
- responseBadRequest(c, err)
- return
- }
- slotRanges := make([]metadata.SlotRange, len(payload.Slots))
- for i, slot := range payload.Slots {
- slotRange, err := metadata.ParseSlotRange(slot)
- if err != nil {
- responseBadRequest(c, err)
- return
- }
- slotRanges[i] = *slotRange
- }
-
- storage := handler.storage
- if isAdd {
- err = storage.AddShardSlots(c, ns, cluster, shard, slotRanges)
- } else {
- err = storage.RemoveShardSlots(c, ns, cluster, shard, slotRanges)
- }
- if err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
-
-func (handler *ShardHandler) MigrateSlotData(c *gin.Context) {
- var req MigrateSlotDataRequest
- if err := c.BindJSON(&req); err != nil {
- responseBadRequest(c, err)
- return
- }
- task := &storage.MigrationTask{
- Namespace: c.Param("namespace"),
- Cluster: c.Param("cluster"),
- Source: req.Source,
- Target: req.Target,
- Slot: req.Slot,
- TaskID: util.RandString(16),
- StartTime: time.Now().Unix(),
- }
- migrator, _ := c.MustGet(consts.ContextKeyMigrator).(*migrate.Migrator)
- if err := migrator.AddTask(c, task); err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
-
-func (handler *ShardHandler) MigrateSlotOnly(c *gin.Context) {
- var req MigrateSlotOnlyRequest
- if err := c.BindJSON(&req); err != nil {
- responseBadRequest(c, err)
- return
- }
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- if err := handler.storage.UpdateMigrateSlotInfo(c, ns, cluster, req.Source, req.Target, req.Slots); err != nil {
- responseError(c, err)
- return
- }
- responseOK(c, "ok")
-}
-
-func (handler *ShardHandler) Failover(c *gin.Context) {
- ns := c.Param("namespace")
- cluster := c.Param("cluster")
- shard, err := strconv.Atoi(c.Param("shard"))
- if err != nil {
- responseBadRequest(c, err)
- return
- }
-
- nodes, err := handler.storage.ListNodes(c, ns, cluster, shard)
- if err != nil {
- return
- }
- if len(nodes) <= 1 {
- responseBadRequest(c, errors.New("no node to be failover"))
- return
- }
- var failoverNode *metadata.NodeInfo
- for i, node := range nodes {
- if node.Role == metadata.RoleMaster {
- failoverNode = &nodes[i]
- break
- }
- }
- if failoverNode == nil {
- responseBadRequest(c, metadata.ErrEntryNoExists)
- return
- }
-
- f, _ := c.MustGet(consts.ContextKeyFailover).(*failover.FailOver)
- err = f.AddNode(ns, cluster, shard, *failoverNode, failover.ManualType)
- if err != nil {
- responseBadRequest(c, err)
- return
- }
- responseOK(c, "ok")
-}
diff --git a/server/shard_test.go b/server/shard_test.go
deleted file mode 100644
index aeb1828..0000000
--- a/server/shard_test.go
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package server
-
-import (
- "fmt"
- "net/http"
- "testing"
-
- "github.com/google/uuid"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/stretchr/testify/require"
-)
-
-func TestShard(t *testing.T) {
- ns := uuid.NewString()
- clusterName := uuid.NewString()
- shardURI := fmt.Sprintf("%s/%s/clusters/%s/shards", "/api/v1/namespaces", ns, clusterName)
-
- createTestNamespace(t, ns)
- createTestCluster(t, ns, clusterName)
-
- t.Run("Create Shard", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- var req struct {
- Nodes []string `json:"nodes"`
- }
- req.Nodes = []string{"1.1.1.4"}
- newRequest().
- Post(shardURI).
- JSON(req).
- Expect(t).
- Status(http.StatusCreated).
- End()
- })
-
- t.Run("Get shard", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Shard metadata.Shard `json:"shard"`
- }
- }
- newRequest().
- Get(shardURI + "/3").
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Equal(t, 1, len(result.Data.Shard.Nodes))
- })
-
- t.Run("List shards", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
-
- var result struct {
- Data struct {
- Shards []metadata.Shard `json:"shards"`
- }
- }
- newRequest().
- Get(shardURI).
- Expect(t).
- Status(http.StatusOK).
- End().
- JSON(&result)
- require.Equal(t, 4, len(result.Data.Shards))
- })
-
- t.Run("Delete shard", func(t *testing.T) {
- server, newRequest := newTestServer()
- require.NotEmpty(t, server)
- newRequest().
- Delete(shardURI + "/3").
- Expect(t).
- Status(http.StatusOK).
- End()
- })
-}
diff --git a/storage/failover.go b/storage/failover.go
deleted file mode 100644
index 8b293e9..0000000
--- a/storage/failover.go
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "encoding/json"
-
- "github.com/apache/kvrocks-controller/metadata"
-
- "golang.org/x/net/context"
-)
-
-type FailoverTask struct {
- Namespace string `json:"namespace"`
- Cluster string `json:"cluster"`
- ShardIdx int `json:"shard_idx"`
- Node metadata.NodeInfo `json:"node"`
- Type int `json:"type"`
- ProbeCount int `json:"probe_count"`
-
- QueuedTime int64 `json:"pending_time"`
- StartTime int64 `json:"start_time"`
- FinishTime int64 `json:"finish_time"`
-
- Status int `json:"status"`
- Err string `json:"error"`
-}
-
-func (s *Storage) UpdateFailOverTask(ctx context.Context, task *FailoverTask) error {
- taskData, err := json.Marshal(task)
- if err != nil {
- return err
- }
- return s.persist.Set(ctx, buildFailOverKey(task.Namespace, task.Cluster), taskData)
-}
-
-func (s *Storage) GetFailOverTask(ctx context.Context, ns, cluster string) (*FailoverTask, error) {
- taskKey := buildFailOverKey(ns, cluster)
- value, err := s.persist.Get(ctx, taskKey)
- if err != nil {
- return nil, err
- }
- if len(value) == 0 {
- return nil, nil // nolint
- }
- var task FailoverTask
- if err := json.Unmarshal(value, &task); err != nil {
- return nil, err
- }
- return &task, nil
-}
diff --git a/storage/migrate.go b/storage/migrate.go
deleted file mode 100644
index c04bd83..0000000
--- a/storage/migrate.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "encoding/json"
- "errors"
-
- "github.com/apache/kvrocks-controller/metadata"
-
- "golang.org/x/net/context"
-)
-
-var (
- errNilMigrateTask = errors.New("nil migrate task")
-)
-
-type MigrationTask struct {
- Namespace string `json:"namespace"`
- Cluster string `json:"cluster"`
- TaskID string `json:"task_id"`
- Source int `json:"source"`
- SourceNode *metadata.NodeInfo `json:"source_node"`
- Target int `json:"target"`
- TargetNode *metadata.NodeInfo `json:"target_node"`
- Slot int `json:"slot"`
-
- StartTime int64 `json:"start_time"`
- FinishTime int64 `json:"finish_time"`
-
- Status int `json:"status"`
- ErrorDetail string `json:"error_detail"`
-}
-
-func (s *Storage) AddMigratingTask(ctx context.Context, task *MigrationTask) error {
- taskData, err := json.Marshal(task)
- if err != nil {
- return err
- }
- return s.persist.Set(ctx, buildMigratingKeyPrefix(task.Namespace, task.Cluster), taskData)
-}
-
-func (s *Storage) GetMigratingTask(ctx context.Context, ns, cluster string) (*MigrationTask, error) {
- taskKey := buildMigratingKeyPrefix(ns, cluster)
- value, err := s.persist.Get(ctx, taskKey)
- if err != nil && !errors.Is(err, metadata.ErrEntryNoExists) {
- return nil, err
- }
- if len(value) == 0 {
- return nil, nil // nolint
- }
- var task MigrationTask
- if err := json.Unmarshal(value, &task); err != nil {
- return nil, err
- }
- return &task, nil
-}
-
-func (s *Storage) RemoveMigratingTask(ctx context.Context, ns, cluster string) error {
- taskKey := buildMigratingKeyPrefix(ns, cluster)
- return s.persist.Delete(ctx, taskKey)
-}
diff --git a/storage/node.go b/storage/node.go
deleted file mode 100644
index 9c6862c..0000000
--- a/storage/node.go
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "errors"
- "fmt"
- "strconv"
- "strings"
-
- "golang.org/x/net/context"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/util"
-)
-
-// ListNodes return the list of nodes under the specified shard
-func (s *Storage) ListNodes(ctx context.Context, ns, cluster string, shardIdx int) ([]metadata.NodeInfo, error) {
- shard, err := s.getShard(ctx, ns, cluster, shardIdx)
- if err != nil {
- return nil, fmt.Errorf("get shard: %w", err)
- }
- return shard.Nodes, nil
-}
-
-// GetMasterNode return the master of node under the specified shard
-func (s *Storage) GetMasterNode(ctx context.Context, ns, cluster string, shardIdx int) (metadata.NodeInfo, error) {
- nodes, err := s.ListNodes(ctx, ns, cluster, shardIdx)
- if err != nil {
- return metadata.NodeInfo{}, err
- }
-
- for _, node := range nodes {
- if node.Role == metadata.RoleMaster {
- return node, nil
- }
- }
- return metadata.NodeInfo{}, metadata.ErrEntryNoExists
-}
-
-// CreateNode add a node under the specified shard
-func (s *Storage) CreateNode(ctx context.Context, ns, cluster string, shardIdx int, node *metadata.NodeInfo) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return fmt.Errorf("get cluster: %w", err)
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- for _, shard := range clusterInfo.Shards {
- if len(shard.Nodes) == 0 {
- continue
- }
- for _, existedNode := range shard.Nodes {
- if existedNode.Addr == node.Addr || existedNode.ID == node.ID {
- return metadata.ErrEntryExisted
- }
- }
- }
-
- shard := clusterInfo.Shards[shardIdx]
- if shard.Nodes == nil {
- shard.Nodes = make([]metadata.NodeInfo, 0)
- }
-
- // NodeRole check
- if len(shard.Nodes) == 0 && !node.IsMaster() {
- return errors.New("you MUST add master node first")
- }
- if len(shard.Nodes) != 0 && node.IsMaster() {
- return errors.New("the master node has already added in this shard")
- }
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx].Nodes = append(clusterInfo.Shards[shardIdx].Nodes, *node)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- NodeID: node.ID,
- Type: EventNode,
- Command: CommandCreate,
- })
- return nil
-}
-
-// RemoveNode delete the node from the specified shard
-func (s *Storage) RemoveNode(ctx context.Context, ns, cluster string, shardIdx int, nodeID string) error {
- if len(nodeID) != metadata.NodeIdLen {
- return errors.New("invalid node length")
- }
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return fmt.Errorf("get cluster: %w", err)
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- shard := clusterInfo.Shards[shardIdx]
- if shard.Nodes == nil {
- return metadata.ErrEntryNoExists
- }
- nodeIdx := -1
- for idx, node := range shard.Nodes {
- if strings.HasPrefix(node.ID, nodeID) {
- nodeIdx = idx
- break
- }
- }
- if nodeIdx == -1 {
- return metadata.ErrEntryNoExists
- }
- node := shard.Nodes[nodeIdx]
- if len(shard.SlotRanges) != 0 {
- if len(shard.Nodes) == 1 || node.IsMaster() {
- return errors.New("still some slots in this shard, please migrate them first")
- }
- } else {
- if node.IsMaster() && len(shard.Nodes) > 1 {
- return errors.New("please remove slave shards first")
- }
- }
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx].Nodes = append(clusterInfo.Shards[shardIdx].Nodes[:nodeIdx], clusterInfo.Shards[shardIdx].Nodes[nodeIdx+1:]...)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- NodeID: node.ID,
- Type: EventNode,
- Command: CommandRemove,
- })
- return nil
-}
-
-// PromoteNewMaster delete the master node from the specified shard
-func (s *Storage) PromoteNewMaster(ctx context.Context, ns, cluster string, shardIdx int, oldMasterNodeID string) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return fmt.Errorf("get cluster: %w", err)
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- shard := clusterInfo.Shards[shardIdx]
- if shard.Nodes == nil || len(shard.Nodes) == 1 {
- return metadata.ErrShardNoReplica
- }
-
- var oldMasterNode *metadata.NodeInfo
- for idx, node := range shard.Nodes {
- if node.ID == oldMasterNodeID {
- oldMasterNode = &shard.Nodes[idx]
- break
- }
- }
- if oldMasterNode == nil {
- return fmt.Errorf("master node: %s not found in shard: %d", oldMasterNodeID, shardIdx)
- }
- if oldMasterNode.Role != metadata.RoleMaster {
- return errors.New("node %s role is not master")
- }
-
- var (
- oldMasterNodeIndex = -1
- fastestSlaveIndex = -1
- maxReplOffset uint64
- )
- for idx, node := range shard.Nodes {
- if node.ID == oldMasterNodeID {
- oldMasterNodeIndex = idx
- continue
- }
- nodeInfo, err := util.NodeInfoCmd(ctx, &node)
- if err != nil {
- continue
- }
- offsetNodeStr := nodeInfo.SlaveReplication.SlaveReplOffset
- if len(offsetNodeStr) == 0 {
- continue
- }
- offsetNode, _ := strconv.ParseUint(offsetNodeStr, 10, 64)
- if offsetNode >= maxReplOffset {
- maxReplOffset = offsetNode
- fastestSlaveIndex = idx
- }
- }
- if oldMasterNodeIndex == -1 {
- return metadata.ErrEntryNoExists
- }
- if fastestSlaveIndex == -1 {
- return metadata.ErrShardNoMatchPromoteNode
- }
-
- shard.Nodes[fastestSlaveIndex].Role = metadata.RoleMaster
- shard.Nodes[oldMasterNodeIndex].Role = metadata.RoleSlave
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx] = shard
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- NodeID: shard.Nodes[fastestSlaveIndex].ID,
- Type: EventNode,
- Command: CommandRemove,
- })
- return nil
-}
-
-// UpdateNode update exists node under the specified shard
-func (s *Storage) UpdateNode(ctx context.Context, ns, cluster string, shardIdx int, node *metadata.NodeInfo) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return fmt.Errorf("get cluster: %w", err)
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- shard := clusterInfo.Shards[shardIdx]
- if shard.Nodes == nil {
- return metadata.ErrEntryNoExists
- }
- // TODO: check the role
- for idx, existedNode := range shard.Nodes {
- if existedNode.ID == node.ID {
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx].Nodes[idx] = *node
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- NodeID: node.ID,
- Type: EventNode,
- Command: CommandUpdate,
- })
- return nil
- }
- }
- return metadata.ErrEntryNoExists
-}
diff --git a/storage/persistence/persistence.go b/storage/persistence/persistence.go
deleted file mode 100644
index 861d6a4..0000000
--- a/storage/persistence/persistence.go
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package persistence
-
-import (
- "context"
-)
-
-type Entry struct {
- Key string
- Value []byte
-}
-
-type Persistence interface {
- ID() string
- Leader() string
- LeaderChange() <-chan bool
- IsReady(ctx context.Context) bool
-
- Get(ctx context.Context, key string) ([]byte, error)
- Exists(ctx context.Context, key string) (bool, error)
- Set(ctx context.Context, key string, value []byte) error
- Delete(ctx context.Context, key string) error
- List(ctx context.Context, prefix string) ([]Entry, error)
-
- Close() error
-}
diff --git a/storage/setup_test.go b/storage/setup_test.go
deleted file mode 100644
index 1a90a9e..0000000
--- a/storage/setup_test.go
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "os"
- "testing"
- "time"
-
- clientv3 "go.etcd.io/etcd/client/v3"
-)
-
-var testEtcdClient *clientv3.Client
-
-func setup() (err error) {
- testEtcdClient, err = clientv3.New(clientv3.Config{
- Endpoints: []string{"0.0.0.0:2379"},
- DialTimeout: 5 * time.Second,
- })
- return err
-}
-
-func TestMain(m *testing.M) {
- if err := setup(); err != nil {
- panic("Failed to setup the etcd client: " + err.Error())
- }
- os.Exit(m.Run())
-}
diff --git a/storage/shard.go b/storage/shard.go
deleted file mode 100644
index fb8442d..0000000
--- a/storage/shard.go
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "errors"
- "fmt"
-
- "golang.org/x/net/context"
-
- "github.com/apache/kvrocks-controller/metadata"
-)
-
-func (s *Storage) ListShard(ctx context.Context, ns, cluster string) ([]metadata.Shard, error) {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return nil, err
- }
- return clusterInfo.Shards, nil
-}
-
-func (s *Storage) GetShard(ctx context.Context, ns, cluster string, shardIdx int) (*metadata.Shard, error) {
- return s.getShard(ctx, ns, cluster, shardIdx)
-}
-
-func (s *Storage) getShard(ctx context.Context, ns, cluster string, shardIdx int) (*metadata.Shard, error) {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return nil, err
- }
- if clusterInfo.Shards == nil {
- return nil, metadata.ErrEntryNoExists
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return nil, metadata.ErrIndexOutOfRange
- }
- return &clusterInfo.Shards[shardIdx], nil
-}
-
-// CreateShard add a shard under the specified cluster
-func (s *Storage) CreateShard(ctx context.Context, ns, cluster string, shard *metadata.Shard) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return err
- }
- clusterInfo.Version++
- clusterInfo.Shards = append(clusterInfo.Shards, *shard)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: len(clusterInfo.Shards) - 1,
- Type: EventShard,
- Command: CommandCreate,
- })
- return nil
-}
-
-// RemoveShard delete the shard under the specified cluster
-func (s *Storage) RemoveShard(ctx context.Context, ns, cluster string, shardIdx int) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return err
- }
- if shardIdx >= len(clusterInfo.Shards) || shardIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- shard := clusterInfo.Shards[shardIdx]
- if len(shard.SlotRanges) > 0 {
- return fmt.Errorf("need to delete all slots before removing shard")
- }
- clusterInfo.Version++
- clusterInfo.Shards = append(clusterInfo.Shards[:shardIdx], clusterInfo.Shards[shardIdx+1:]...)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- Type: EventShard,
- Command: CommandRemove,
- })
- return nil
-}
-
-// HasSlot return an indicator whether the slot under the specified Shard
-func (s *Storage) HasSlot(ctx context.Context, ns, cluster string, shardIdx, slot int) (bool, error) {
- shard, err := s.GetShard(ctx, ns, cluster, shardIdx)
- if err != nil {
- return false, err
- }
- for _, slots := range shard.SlotRanges {
- if slot >= slots.Start && slot <= slots.Stop {
- return true, nil
- }
- }
- return false, nil
-}
-
-// AddShardSlots add slotRanges to the specified shard under the specified cluster
-func (s *Storage) AddShardSlots(ctx context.Context, ns, cluster string, shardIdx int, slotRanges []metadata.SlotRange) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return err
- }
- shard, err := s.getShard(ctx, ns, cluster, shardIdx)
- if err != nil {
- return fmt.Errorf("get shard: %w", err)
- }
- if len(shard.Nodes) == 0 {
- return errors.New("the shard was empty, please add Shards first")
- }
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx].SlotRanges = metadata.MergeSlotRanges(shard.SlotRanges, slotRanges)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- Type: EventShard,
- Command: CommandAddSlots,
- })
- return nil
-}
-
-func (s *Storage) RemoveShardSlots(ctx context.Context, ns, cluster string, shardIdx int, slotRanges []metadata.SlotRange) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return err
- }
- shard, err := s.getShard(ctx, ns, cluster, shardIdx)
- if err != nil {
- return fmt.Errorf("get shard: %w", err)
- }
- clusterInfo.Version++
- clusterInfo.Shards[shardIdx].SlotRanges = metadata.RemoveSlotRanges(shard.SlotRanges, slotRanges)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: shardIdx,
- Type: EventShard,
- Command: CommandRemoveSlots,
- })
- return nil
-}
-
-func (s *Storage) UpdateMigrateSlotInfo(ctx context.Context, ns,
- cluster string, sourceIdx, targetIdx int, slotRanges []metadata.SlotRange) error {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return err
- }
- if clusterInfo.Shards == nil {
- return metadata.ErrEntryNoExists
- }
- if sourceIdx >= len(clusterInfo.Shards) || sourceIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
- if targetIdx >= len(clusterInfo.Shards) || targetIdx < 0 {
- return metadata.ErrIndexOutOfRange
- }
-
- sourceShard := clusterInfo.Shards[sourceIdx]
- targetShard := clusterInfo.Shards[targetIdx]
- clusterInfo.Version++
- clusterInfo.Shards[sourceIdx].SlotRanges = metadata.RemoveSlotRanges(sourceShard.SlotRanges, slotRanges)
- clusterInfo.Shards[targetIdx].SlotRanges = metadata.MergeSlotRanges(targetShard.SlotRanges, slotRanges)
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Shard: sourceIdx,
- Type: EventShard,
- Command: CommandMigrateSlots,
- })
- return nil
-}
diff --git a/storage/storage.go b/storage/storage.go
deleted file mode 100644
index 5039646..0000000
--- a/storage/storage.go
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "time"
-
- "github.com/apache/kvrocks-controller/storage/persistence"
-
- "github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/metadata"
- "go.uber.org/zap"
-)
-
-var (
- ErrNoLeaderOrNotReady = errors.New("the current node role isn't leader or the state is NOT ready")
-)
-
-type Storage struct {
- persist persistence.Persistence
-
- eventNotifyCh chan Event
- quitCh chan struct{}
-}
-
-func NewStorage(persist persistence.Persistence) (*Storage, error) {
- return &Storage{
- persist: persist,
- eventNotifyCh: make(chan Event, 100),
- quitCh: make(chan struct{}),
- }, nil
-}
-
-func (s *Storage) IsReady() bool {
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
- return s.persist.IsReady(ctx)
-}
-
-// ListNamespace return the list of name of all namespaces
-func (s *Storage) ListNamespace(ctx context.Context) ([]string, error) {
- entries, err := s.persist.List(ctx, MetadataPrefix)
- if err != nil {
- return nil, err
- }
- keys := make([]string, len(entries))
- for i, entry := range entries {
- keys[i] = entry.Key
- }
- return keys, nil
-}
-
-// IsNamespaceExists return an indicator whether the specified namespace exists
-func (s *Storage) IsNamespaceExists(ctx context.Context, ns string) (bool, error) {
- return s.persist.Exists(ctx, appendNamespacePrefix(ns))
-}
-
-// CreateNamespace will create a namespace for clusters
-func (s *Storage) CreateNamespace(ctx context.Context, ns string) error {
- if has, _ := s.IsNamespaceExists(ctx, ns); has {
- return metadata.ErrEntryExisted
- }
- if err := s.persist.Set(ctx, appendNamespacePrefix(ns), []byte(ns)); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Type: EventNamespace,
- Command: CommandCreate,
- })
- return nil
-}
-
-// RemoveNamespace delete the specified namespace from storage
-func (s *Storage) RemoveNamespace(ctx context.Context, ns string) error {
- if has, _ := s.IsNamespaceExists(ctx, ns); !has {
- return metadata.ErrEntryNoExists
- }
- clusters, err := s.ListCluster(ctx, ns)
- if err != nil {
- return err
- }
- if len(clusters) != 0 {
- return errors.New("namespace wasn't empty, please remove clusters first")
- }
- if err := s.persist.Delete(ctx, appendNamespacePrefix(ns)); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Type: EventNamespace,
- Command: CommandRemove,
- })
- return nil
-}
-
-// ListCluster return the list of name of cluster under the specified namespace
-func (s *Storage) ListCluster(ctx context.Context, ns string) ([]string, error) {
- entries, err := s.persist.List(ctx, buildClusterPrefix(ns))
- if err != nil {
- return nil, err
- }
- keys := make([]string, len(entries))
- for i, entry := range entries {
- keys[i] = entry.Key
- }
- return keys, nil
-}
-
-func (s *Storage) IsClusterExists(ctx context.Context, ns, cluster string) (bool, error) {
- return s.persist.Exists(ctx, buildClusterKey(ns, cluster))
-}
-
-func (s *Storage) GetClusterInfo(ctx context.Context, ns, cluster string) (*metadata.Cluster, error) {
- value, err := s.persist.Get(ctx, buildClusterKey(ns, cluster))
- if err != nil {
- return nil, err
- }
- var clusterInfo metadata.Cluster
- if err = json.Unmarshal(value, &clusterInfo); err != nil {
- return nil, err
- }
- return &clusterInfo, nil
-}
-
-func (s *Storage) ClusterNodesCounts(ctx context.Context, ns, cluster string) (int, error) {
- clusterInfo, err := s.GetClusterInfo(ctx, ns, cluster)
- if err != nil {
- return -1, err
- }
- count := 0
- for _, shard := range clusterInfo.Shards {
- count += len(shard.Nodes)
- }
- return count, nil
-}
-
-// UpdateCluster update the Name to storage under the specified namespace
-func (s *Storage) UpdateCluster(ctx context.Context, ns string, clusterInfo *metadata.Cluster) error {
- return s.updateCluster(ctx, ns, clusterInfo)
-}
-
-// updateCluster is goroutine unsafe of UpdateCluster
-// assumption caller has hold the lock
-func (s *Storage) updateCluster(ctx context.Context, ns string, clusterInfo *metadata.Cluster) error {
- if len(clusterInfo.Shards) == 0 {
- return errors.New("required at least one shard")
- }
- value, err := json.Marshal(clusterInfo)
- if err != nil {
- return err
- }
- return s.persist.Set(ctx, buildClusterKey(ns, clusterInfo.Name), value)
-}
-
-func (s *Storage) CreateCluster(ctx context.Context, ns string, clusterInfo *metadata.Cluster) error {
- if exists, _ := s.IsClusterExists(ctx, ns, clusterInfo.Name); exists {
- return metadata.ErrEntryExisted
- }
- if err := s.updateCluster(ctx, ns, clusterInfo); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: clusterInfo.Name,
- Type: EventCluster,
- Command: CommandCreate,
- })
- return nil
-}
-
-func (s *Storage) RemoveCluster(ctx context.Context, ns, cluster string) error {
- if exists, _ := s.IsClusterExists(ctx, ns, cluster); !exists {
- return metadata.ErrEntryNoExists
- }
- if err := s.persist.Delete(ctx, buildClusterKey(ns, cluster)); err != nil {
- return err
- }
- s.EmitEvent(Event{
- Namespace: ns,
- Cluster: cluster,
- Type: EventCluster,
- Command: CommandRemove,
- })
- return nil
-}
-
-func (s *Storage) Load(ctx context.Context) error {
- namespaces, err := s.ListNamespace(ctx)
- if err != nil {
- return err
- }
- for _, namespace := range namespaces {
- clusters, err := s.ListCluster(ctx, namespace)
- if err != nil {
- return fmt.Errorf("list cluster in namespace[%s] err: %w", namespace, err)
- }
- for _, cluster := range clusters {
- _, err := s.GetClusterInfo(ctx, namespace, cluster)
- if errors.Is(err, metadata.ErrEntryNoExists) {
- logger.Get().With(
- zap.Error(err),
- zap.String("cluster", cluster),
- ).Warn("Can't load the cluster from storage")
- continue
- }
- if err != nil {
- return fmt.Errorf("get cluster[%s] err: %w", cluster, err)
- }
- }
- }
- return nil
-}
-
-func (s *Storage) Notify() <-chan Event {
- return s.eventNotifyCh
-}
-
-func (s *Storage) EmitEvent(event Event) {
- s.eventNotifyCh <- event
-}
-
-func (s *Storage) LeaderChange() <-chan bool {
- return s.persist.LeaderChange()
-}
-
-func (s *Storage) IsLeader() bool {
- return s.persist.Leader() == s.persist.ID()
-}
-
-func (s *Storage) Leader() string {
- return s.persist.Leader()
-}
-
-func (s *Storage) Close() error {
- return s.persist.Close()
-}
-
-func (s *Storage) Stop() error {
- return nil
-}
diff --git a/storage/storage_test.go b/storage/storage_test.go
deleted file mode 100644
index d912f1d..0000000
--- a/storage/storage_test.go
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package storage
-
-import (
- "context"
- "sort"
- "testing"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage/persistence/etcd"
- "github.com/apache/kvrocks-controller/util"
- "github.com/stretchr/testify/require"
-)
-
-var storage *Storage
-
-func requireStorage(t *testing.T) {
- if storage != nil {
- return
- }
- e, err := etcd.New("test-storage", &etcd.Config{Addrs: []string{"127.0.0.1:2379"}})
- require.NoError(t, err)
- storage, err = NewStorage(e)
- require.NoError(t, err)
-}
-
-func TestStorage_Namespace(t *testing.T) {
- requireStorage(t)
-
- ctx := context.Background()
- namespaces := []string{util.RandString(40), util.RandString(40), util.RandString(40)}
- sort.Strings(namespaces)
- for _, namespace := range namespaces {
- require.NoError(t, storage.CreateNamespace(ctx, namespace))
- }
-
- for _, namespace := range namespaces {
- exists, err := storage.IsNamespaceExists(ctx, namespace)
- require.NoError(t, err)
- require.True(t, exists)
- }
-
- require.Equal(t, metadata.ErrEntryExisted, storage.CreateNamespace(ctx, namespaces[0]))
-
- gotNamespaces, err := storage.ListNamespace(ctx)
- require.NoError(t, err)
-
- for _, namespace := range namespaces {
- require.Contains(t, gotNamespaces, namespace)
- require.NoError(t, storage.RemoveNamespace(ctx, namespace))
- }
-}
-
-func TestStorage_Cluster(t *testing.T) {
- requireStorage(t)
- ns := util.RandString(10)
- clusterName := util.RandString(10)
- ctx := context.Background()
-
- newClusterInfo := &metadata.Cluster{
- Name: clusterName,
- Shards: []metadata.Shard{
- {
- Nodes: []metadata.NodeInfo{
- {ID: util.RandString(40), Addr: "1.1.1.1:6379", Role: metadata.RoleMaster},
- },
- SlotRanges: []metadata.SlotRange{
- {Start: 0, Stop: 5000},
- },
- },
- },
- }
- err := storage.CreateCluster(ctx, ns, newClusterInfo)
-
- require.NoError(t, err)
- exists, err := storage.IsClusterExists(ctx, ns, clusterName)
- require.NoError(t, err)
- require.True(t, exists)
- clusterInfo, err := storage.GetClusterInfo(ctx, ns, clusterName)
- require.Equal(t, newClusterInfo, clusterInfo)
- require.NoError(t, err)
- require.NoError(t, storage.RemoveCluster(ctx, ns, clusterName))
- exists, err = storage.IsClusterExists(ctx, ns, clusterName)
- require.NoError(t, err)
- require.False(t, exists)
-}
diff --git a/store/cluster.go b/store/cluster.go
new file mode 100644
index 0000000..75f6cef
--- /dev/null
+++ b/store/cluster.go
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+
+ "go.uber.org/atomic"
+
+ "github.com/apache/kvrocks-controller/consts"
+)
+
+type Cluster struct {
+ Name string `json:"name"`
+ Version atomic.Int64 `json:"version"`
+ Shards []*Shard `json:"shards"`
+}
+
+func NewCluster(name string, nodes []string, replicas int) (*Cluster, error) {
+ if len(nodes) == 0 {
+ return nil, errors.New("cluster nodes should NOT be empty")
+ }
+ if replicas < 0 {
+ return nil, errors.New("replicas should NOT be less than 0")
+ }
+ if replicas == 0 {
+ replicas = 1
+ }
+ if len(nodes)%replicas != 0 {
+ return nil, errors.New("cluster nodes should be divisible by replicas")
+ }
+ shardCount := len(nodes) / replicas
+ shards := make([]*Shard, 0)
+ slotRanges := SpiltSlotRange(shardCount)
+ for i := 0; i < shardCount; i++ {
+ shard := NewShard()
+ shard.Nodes = make([]Node, 0)
+ for j := 0; j < replicas; j++ {
+ addr := nodes[i*replicas+j]
+ role := RoleMaster
+ if j != 0 {
+ role = RoleSlave
+ }
+ node := NewClusterNode(addr, "")
+ node.SetRole(role)
+ shard.Nodes = append(shard.Nodes, node)
+ }
+ shard.SlotRanges = append(shard.SlotRanges, slotRanges[i])
+ shards = append(shards, shard)
+ }
+
+ cluster := &Cluster{Name: name, Shards: shards}
+ cluster.Version.Store(1)
+ return cluster, nil
+}
+
+// SetPassword will set the password for all nodes in the cluster.
+func (cluster *Cluster) SetPassword(password string) {
+ for i := 0; i < len(cluster.Shards); i++ {
+ for j := 0; j < len(cluster.Shards[i].Nodes); j++ {
+ cluster.Shards[i].Nodes[j].SetPassword(password)
+ }
+ }
+}
+
+func (cluster *Cluster) ToSlotString() (string, error) {
+ var builder strings.Builder
+ for i, shard := range cluster.Shards {
+ shardSlotsString, err := shard.ToSlotsString()
+ if err != nil {
+ return "", fmt.Errorf("found err at shard[%d]: %w", i, err)
+ }
+ builder.WriteString(shardSlotsString)
+ }
+ return builder.String(), nil
+}
+
+func (cluster *Cluster) GetShard(shardIndex int) (*Shard, error) {
+ if shardIndex < 0 || shardIndex >= len(cluster.Shards) {
+ return nil, consts.ErrIndexOutOfRange
+ }
+ return cluster.Shards[shardIndex], nil
+}
+
+func (cluster *Cluster) AddNode(shardIndex int, addr, role, password string) error {
+ if shardIndex < 0 || shardIndex >= len(cluster.Shards) {
+ return consts.ErrIndexOutOfRange
+ }
+ return cluster.Shards[shardIndex].addNode(addr, role, password)
+}
+
+func (cluster *Cluster) RemoveNode(shardIndex int, nodeID string) error {
+ if shardIndex < 0 || shardIndex >= len(cluster.Shards) {
+ return consts.ErrIndexOutOfRange
+ }
+ return cluster.Shards[shardIndex].removeNode(nodeID)
+}
+
+func (cluster *Cluster) PromoteNewMaster(ctx context.Context,
+ shardIdx int, masterNodeID, preferredNodeID string) (string, error) {
+ shard, err := cluster.GetShard(shardIdx)
+ if err != nil {
+ return "", err
+ }
+ newMasterNodeID, err := shard.promoteNewMaster(ctx, masterNodeID, preferredNodeID)
+ if err != nil {
+ return "", err
+ }
+ cluster.Shards[shardIdx] = shard
+ return newMasterNodeID, nil
+}
+
+func (cluster *Cluster) SyncToNodes(ctx context.Context) error {
+ for i := 0; i < len(cluster.Shards); i++ {
+ for _, node := range cluster.Shards[i].Nodes {
+ if err := node.SyncClusterInfo(ctx, cluster); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (cluster *Cluster) Reset(ctx context.Context) error {
+ for i := 0; i < len(cluster.Shards); i++ {
+ for _, node := range cluster.Shards[i].Nodes {
+ if err := node.Reset(ctx); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func (cluster *Cluster) findShardIndexBySlot(slot int) (int, error) {
+ if slot < 0 || slot >= MaxSlotID {
+ return -1, consts.ErrSlotOutOfRange
+ }
+ sourceShardIdx := -1
+ for i := 0; i < len(cluster.Shards); i++ {
+ slotRanges := cluster.Shards[i].SlotRanges
+ for _, slotRange := range slotRanges {
+ if slotRange.Contains(slot) {
+ sourceShardIdx = i
+ break
+ }
+ }
+ }
+ if sourceShardIdx == -1 {
+ return -1, consts.ErrSlotNotBelongToAnyShard
+ }
+ return sourceShardIdx, nil
+}
+
+func (cluster *Cluster) MigrateSlot(ctx context.Context, slot int, targetShardIdx int, slotOnly bool) error {
+ if targetShardIdx < 0 || targetShardIdx >= len(cluster.Shards) {
+ return consts.ErrIndexOutOfRange
+ }
+ sourceShardIdx, err := cluster.findShardIndexBySlot(slot)
+ if err != nil {
+ return err
+ }
+ if sourceShardIdx == targetShardIdx {
+ return consts.ErrShardIsSame
+ }
+ if slotOnly {
+ migrateSlot := SlotRange{Start: slot, Stop: slot}
+ cluster.Shards[sourceShardIdx].SlotRanges = RemoveSlotRanges(cluster.Shards[sourceShardIdx].SlotRanges, []SlotRange{migrateSlot})
+ cluster.Shards[targetShardIdx].SlotRanges = MergeSlotRanges(cluster.Shards[targetShardIdx].SlotRanges, []SlotRange{migrateSlot})
+ return nil
+ }
+
+ if cluster.Shards[sourceShardIdx].IsMigrating() || cluster.Shards[targetShardIdx].IsMigrating() {
+ return consts.ErrShardSlotIsMigrating
+ }
+ // Send the migration command to the source node
+ sourceMasterNode := cluster.Shards[sourceShardIdx].GetMasterNode()
+ if sourceMasterNode == nil {
+ return consts.ErrNotFound
+ }
+ targetNodeID := cluster.Shards[targetShardIdx].GetMasterNode().ID()
+ if err := sourceMasterNode.MigrateSlot(ctx, slot, targetNodeID); err != nil {
+ return err
+ }
+
+ // Will start the data migration in the background
+ cluster.Shards[sourceShardIdx].MigratingSlot = slot
+ cluster.Shards[sourceShardIdx].TargetShardIndex = targetShardIdx
+ return nil
+}
+
+func (c *Cluster) SetSlot(ctx context.Context, slot int, targetNodeID string) error {
+ version := c.Version.Inc()
+ for i := 0; i < len(c.Shards); i++ {
+ for _, node := range c.Shards[i].Nodes {
+ clusterNode, ok := node.(*ClusterNode)
+ if !ok {
+ continue
+ }
+ err := clusterNode.GetClient().Do(ctx, "CLUSTERX", "SETSLOT", slot, "NODE", targetNodeID, version).Err()
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// ParseCluster will parse the cluster string into cluster topology.
+func ParseCluster(clusterStr string) (*Cluster, error) {
+ if len(clusterStr) == 0 {
+ return nil, errors.New("cluster nodes string error")
+ }
+ nodeStrings := strings.Split(clusterStr, "\n")
+ if len(nodeStrings) == 0 {
+ return nil, errors.New("cluster nodes string parser error")
+ }
+
+ var clusterVer int64 = -1
+ var shards Shards
+ slaveNodes := make(map[string][]Node)
+ for _, nodeString := range nodeStrings {
+ fields := strings.Split(nodeString, " ")
+ if len(fields) < 7 {
+ return nil, fmt.Errorf("require at least 7 fields, node info[%s]", nodeString)
+ }
+ node := &ClusterNode{
+ id: fields[0],
+ addr: strings.Split(fields[1], "@")[0],
+ }
+
+ if strings.Contains(fields[2], ",") {
+ node.role = strings.Split(fields[2], ",")[1]
+ } else {
+ node.role = fields[2]
+ }
+
+ var err error
+ clusterVer, err = strconv.ParseInt(fields[6], 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("node version error, node info[%q]", nodeString)
+ }
+
+ if node.role == RoleMaster {
+ if len(fields) < 9 {
+ return nil, fmt.Errorf("master node element less 9, node info[%q]", nodeString)
+ }
+ slots, err := ParseSlotRange(fields[8])
+ if err != nil {
+ return nil, fmt.Errorf("master node parser slot error, node info[%q]", nodeString)
+ }
+ shard := NewShard()
+ shard.Nodes = append(shard.Nodes, node)
+ shard.SlotRanges = append(shard.SlotRanges, *slots)
+ shards = append(shards, shard)
+ } else if node.role == RoleSlave {
+ slaveNodes[fields[3]] = append(slaveNodes[fields[3]], node)
+ } else {
+ return nil, fmt.Errorf("node role error, node info[%q]", nodeString)
+ }
+ }
+ if clusterVer == -1 {
+ return nil, fmt.Errorf("no cluster version, cluster info[%q]", clusterStr)
+ }
+ sort.Sort(shards)
+ for i := 0; i < len(shards); i++ {
+ masterNode := shards[i].Nodes[0]
+ shards[i].Nodes = append(shards[i].Nodes, slaveNodes[masterNode.ID()]...)
+ }
+ return &Cluster{
+ Version: *atomic.NewInt64(clusterVer),
+ Shards: shards,
+ }, nil
+}
diff --git a/storage/helper.go b/store/cluster_mock_node.go
similarity index 50%
rename from storage/helper.go
rename to store/cluster_mock_node.go
index fe7a3b5..35988ae 100644
--- a/storage/helper.go
+++ b/store/cluster_mock_node.go
@@ -17,30 +17,39 @@
* under the License.
*
*/
-package storage
-import (
- "fmt"
-)
+package store
-const MetadataPrefix = "/kvrocks/metadata"
+import "context"
-func appendNamespacePrefix(ns string) string {
- return MetadataPrefix + "/" + ns
+// ClusterMockNode is a mock implementation of the Node interface,
+// it is used for testing purposes.
+type ClusterMockNode struct {
+ *ClusterNode
+
+ Sequence uint64
+}
+
+var _ Node = (*ClusterMockNode)(nil)
+
+func NewClusterMockNode() *ClusterMockNode {
+ return &ClusterMockNode{
+ ClusterNode: NewClusterNode("", ""),
+ }
}
-func buildClusterPrefix(ns string) string {
- return fmt.Sprintf("%s/%s/cluster", MetadataPrefix, ns)
+func (mock *ClusterMockNode) GetClusterNodeInfo(ctx context.Context) (*ClusterNodeInfo, error) {
+ return &ClusterNodeInfo{Sequence: mock.Sequence}, nil
}
-func buildClusterKey(ns, cluster string) string {
- return fmt.Sprintf("%s/%s", buildClusterPrefix(ns), cluster)
+func (mock *ClusterMockNode) GetClusterInfo(ctx context.Context) (*ClusterInfo, error) {
+ return &ClusterInfo{}, nil
}
-func buildMigratingKeyPrefix(ns, cluster string) string {
- return fmt.Sprintf("%s/%s/migrate/doing", buildClusterPrefix(ns), cluster)
+func (mock *ClusterMockNode) SyncClusterInfo(ctx context.Context, cluster *Cluster) error {
+ return nil
}
-func buildFailOverKey(ns, cluster string) string {
- return fmt.Sprintf("%s/%s/failover/queue", buildClusterPrefix(ns), cluster)
+func (mock *ClusterMockNode) Reset(ctx context.Context) error {
+ return nil
}
diff --git a/store/cluster_node.go b/store/cluster_node.go
new file mode 100644
index 0000000..5ee9520
--- /dev/null
+++ b/store/cluster_node.go
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/kvrocks-controller/util"
+
+ "github.com/go-playground/validator/v10"
+ "github.com/go-redis/redis/v8"
+)
+
+const (
+ RoleMaster = "master"
+ RoleSlave = "slave"
+
+ NodeIDLen = 40
+)
+
+const (
+ dialTimeout = 3200 * time.Millisecond
+ readTimeout = 3 * time.Second
+ writeTimeout = 3 * time.Second
+ minIdleConns = 3
+)
+
+var _validator = validator.New()
+
+type Node interface {
+ ID() string
+ Password() string
+ Addr() string
+ IsMaster() bool
+
+ SetRole(string)
+ SetPassword(string)
+
+ Reset(ctx context.Context) error
+ GetClusterNodeInfo(ctx context.Context) (*ClusterNodeInfo, error)
+ GetClusterInfo(ctx context.Context) (*ClusterInfo, error)
+ SyncClusterInfo(ctx context.Context, cluster *Cluster) error
+ MigrateSlot(ctx context.Context, slot int, NodeID string) error
+
+ MarshalJSON() ([]byte, error)
+ UnmarshalJSON(data []byte) error
+}
+
+type ClusterNode struct {
+ mu sync.RWMutex
+ client *redis.Client
+
+ id string
+ addr string
+ role string
+ password string
+ createdAt int64
+}
+
+type ClusterInfo struct {
+ CurrentEpoch int64 `json:"cluster_current_epoch"`
+ MigratingSlot int `json:"migrating_slot"`
+ MigratingState string `json:"migrating_state"`
+}
+
+type ClusterNodeInfo struct {
+ Sequence uint64 `json:"sequence"`
+ Role string `json:"role"`
+}
+
+func NewClusterNode(addr, password string) *ClusterNode {
+ return &ClusterNode{
+ id: util.GenerateNodeID(),
+ addr: addr,
+ password: password,
+ role: RoleMaster,
+ createdAt: time.Now().Unix(),
+ }
+}
+
+func (n *ClusterNode) ID() string {
+ return n.id
+}
+
+func (n *ClusterNode) Password() string {
+ return n.password
+}
+
+func (n *ClusterNode) SetPassword(password string) {
+ n.password = password
+}
+
+func (n *ClusterNode) SetRole(role string) {
+ n.role = role
+}
+
+func (n *ClusterNode) Addr() string {
+ return n.addr
+}
+
+func (n *ClusterNode) Validate() error {
+ if len(n.id) == 0 {
+ return errors.New("node id shouldn't be empty")
+ }
+ if len(n.id) != NodeIDLen {
+ return errors.New("the length of node id must be 40")
+ }
+ if n.role != RoleMaster && n.role != RoleSlave {
+ return errors.New("node role should be 'master' or 'slave'")
+ }
+ return _validator.Struct(n)
+}
+
+func (n *ClusterNode) IsMaster() bool {
+ return n.role == RoleMaster
+}
+
+func (n *ClusterNode) GetClient() *redis.Client {
+ n.mu.RLock()
+ if n.client != nil {
+ n.mu.RUnlock()
+ return n.client
+ }
+ n.mu.RUnlock()
+
+ n.mu.Lock()
+ n.client = redis.NewClient(&redis.Options{
+ Addr: n.addr,
+ Password: n.password,
+ DialTimeout: dialTimeout,
+ ReadTimeout: readTimeout,
+ WriteTimeout: writeTimeout,
+ MaxRetries: -1, // don't retry inside the client
+ MinIdleConns: minIdleConns,
+ })
+ n.mu.Unlock()
+ return n.client
+}
+
+func (n *ClusterNode) CheckClusterMode(ctx context.Context) (int64, error) {
+ clusterInfo, err := n.GetClusterInfo(ctx)
+ if err != nil {
+ if strings.Contains(err.Error(), "cluster is not initialized") {
+ return -1, nil
+ }
+ return -1, fmt.Errorf("error while checking node(%s) cluster mode: %w", n.addr, err)
+ }
+ return clusterInfo.CurrentEpoch, nil
+}
+
+func (n *ClusterNode) GetClusterInfo(ctx context.Context) (*ClusterInfo, error) {
+ infoStr, err := n.GetClient().ClusterInfo(ctx).Result()
+ if err != nil {
+ return nil, err
+ }
+
+ clusterInfo := &ClusterInfo{CurrentEpoch: -1}
+ lines := strings.Split(infoStr, "\r\n")
+ for _, line := range lines {
+ fields := strings.Split(line, ":")
+ if len(fields) != 2 {
+ continue
+ }
+ fields[1] = strings.TrimSpace(fields[1])
+ switch strings.ToLower(strings.TrimSpace(fields[0])) {
+ case "cluster_current_epoch":
+ clusterInfo.CurrentEpoch, err = strconv.ParseInt(fields[1], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ case "migrating_slot":
+ clusterInfo.MigratingSlot, err = strconv.Atoi(fields[1])
+ if err != nil {
+ return nil, err
+ }
+ case "migrating_state":
+ clusterInfo.MigratingState = fields[1]
+ }
+ }
+ return clusterInfo, nil
+}
+
+func (n *ClusterNode) GetClusterNodeInfo(ctx context.Context) (*ClusterNodeInfo, error) {
+ infoStr, err := n.GetClient().Info(ctx).Result()
+ if err != nil {
+ return nil, err
+ }
+
+ clusterNodeInfo := &ClusterNodeInfo{}
+ lines := strings.Split(infoStr, "\r\n")
+ for _, line := range lines {
+ fields := strings.Split(line, ":")
+ if len(fields) != 2 {
+ continue
+ }
+ switch fields[0] {
+ case "sequence":
+ clusterNodeInfo.Sequence, err = strconv.ParseUint(fields[1], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ case "role":
+ clusterNodeInfo.Role = fields[1]
+ }
+ }
+ return clusterNodeInfo, nil
+}
+
+func (n *ClusterNode) GetClusterNodesString(ctx context.Context) (string, error) {
+ clusterNodesStr, err := n.GetClient().ClusterNodes(ctx).Result()
+ if err != nil {
+ return "", err
+ }
+ return strings.TrimRight(clusterNodesStr, "\n"), nil
+}
+
+func (n *ClusterNode) SyncClusterInfo(ctx context.Context, cluster *Cluster) error {
+ clusterStr, err := cluster.ToSlotString()
+ if err != nil {
+ return err
+ }
+ redisCli := n.GetClient()
+ err = redisCli.Do(ctx, "CLUSTERX", "SETNODEID", n.id).Err()
+ if err != nil {
+ return err
+ }
+ return redisCli.Do(ctx, "CLUSTERX", "SETNODES", clusterStr, cluster.Version.Load()).Err()
+}
+
+func (n *ClusterNode) Reset(ctx context.Context) error {
+ return n.GetClient().ClusterResetHard(ctx).Err()
+}
+
+func (n *ClusterNode) MigrateSlot(ctx context.Context, slot int, targetNodeID string) error {
+ return n.GetClient().Do(ctx, "CLUSTERX", "MIGRATE", slot, targetNodeID).Err()
+}
+
+func (n *ClusterNode) MarshalJSON() ([]byte, error) {
+ return json.Marshal(map[string]interface{}{
+ "id": n.id,
+ "addr": n.addr,
+ "role": n.role,
+ "password": n.password,
+ "created_at": n.createdAt,
+ })
+}
+
+func (n *ClusterNode) UnmarshalJSON(bytes []byte) error {
+ var data struct {
+ ID string `json:"id"`
+ Addr string `json:"addr"`
+ Role string `json:"role"`
+ Password string `json:"password"`
+ CreatedAt int64 `json:"created_at"`
+ }
+ if err := json.Unmarshal(bytes, &data); err != nil {
+ return err
+ }
+
+ n.id = data.ID
+ n.addr = data.Addr
+ n.role = data.Role
+ n.password = data.Password
+ n.createdAt = data.CreatedAt
+ return nil
+}
diff --git a/store/cluster_node_test.go b/store/cluster_node_test.go
new file mode 100644
index 0000000..e308651
--- /dev/null
+++ b/store/cluster_node_test.go
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestClusterNode(t *testing.T) {
+ ctx := context.Background()
+ defaultNodeAddr := "127.0.0.1:7770"
+ node := NewClusterNode(defaultNodeAddr, "")
+ redisCli := node.GetClient()
+
+ defer func() {
+ require.NoError(t, redisCli.FlushAll(ctx).Err())
+ require.NoError(t, redisCli.Do(ctx, "COMPACT").Err())
+ require.NoError(t, redisCli.Do(ctx, "CLUSTER", "RESET").Err())
+ }()
+
+ t.Run("Check the cluster mode", func(t *testing.T) {
+ _, err := node.CheckClusterMode(ctx)
+ require.NoError(t, err)
+
+ require.NoError(t, redisCli.Do(ctx, "CLUSTER", "RESET").Err())
+ // set the cluster topology
+ cluster := &Cluster{Shards: Shards{
+ {Nodes: []Node{node}, SlotRanges: []SlotRange{{Start: 0, Stop: 16383}}},
+ }}
+ cluster.Version.Store(1)
+ require.NoError(t, node.SyncClusterInfo(ctx, cluster))
+ clusterInfo, err := node.GetClusterInfo(ctx)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, clusterInfo.CurrentEpoch)
+ })
+
+ t.Run("Check the cluster node info", func(t *testing.T) {
+ require.NoError(t, redisCli.Set(ctx, "foo", "bar", 0).Err())
+ info, err := node.GetClusterNodeInfo(ctx)
+ require.NoError(t, err)
+ require.True(t, info.Sequence > 0)
+ })
+
+ t.Run("Parse the cluster node info", func(t *testing.T) {
+ clusterNodesStr, err := node.GetClusterNodesString(ctx)
+ require.NoError(t, err)
+ clusterNodes, err := ParseCluster(clusterNodesStr)
+ require.NoError(t, err)
+ require.EqualValues(t, 1, clusterNodes.Version.Load())
+ require.Len(t, clusterNodes.Shards, 1)
+ require.Len(t, clusterNodes.Shards[0].Nodes, 1)
+ require.EqualValues(t, defaultNodeAddr, clusterNodes.Shards[0].Nodes[0].Addr())
+ require.EqualValues(t, node.ID(), clusterNodes.Shards[0].Nodes[0].ID())
+ })
+}
+
+func TestNodeInfo_Validate(t *testing.T) {
+ node := &ClusterNode{}
+ require.EqualError(t, node.Validate(), "node id shouldn't be empty")
+ node.id = "1234"
+ require.EqualError(t, node.Validate(), "the length of node id must be 40")
+ node.id = strings.Repeat("1", NodeIDLen)
+ require.EqualError(t, node.Validate(), "node role should be 'master' or 'slave'")
+ node.role = RoleMaster
+ node.addr = "1.2.3.4"
+ require.NoError(t, node.Validate())
+}
diff --git a/store/cluster_shard.go b/store/cluster_shard.go
new file mode 100644
index 0000000..eb7164c
--- /dev/null
+++ b/store/cluster_shard.go
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+
+ "go.uber.org/zap"
+
+ "github.com/apache/kvrocks-controller/logger"
+
+ "github.com/apache/kvrocks-controller/consts"
+)
+
+type Shard struct {
+ Nodes []Node `json:"nodes"`
+ SlotRanges []SlotRange `json:"slot_ranges"`
+ TargetShardIndex int `json:"target_shard_index"`
+ MigratingSlot int `json:"migrating_slot"`
+}
+
+type Shards []*Shard
+
+func (s Shards) Len() int {
+ return len(s)
+}
+func (s Shards) Swap(i, j int) {
+ s[i], s[j] = s[j], s[i]
+}
+func (s Shards) Less(i, j int) bool {
+ return s[i].SlotRanges[0].Start < s[j].SlotRanges[0].Start
+}
+
+func NewShard() *Shard {
+ return &Shard{
+ Nodes: make([]Node, 0),
+ SlotRanges: make([]SlotRange, 0),
+ MigratingSlot: -1,
+ TargetShardIndex: -1,
+ }
+}
+
+func (shard *Shard) ClearMigrateState() {
+ shard.MigratingSlot = -1
+ shard.TargetShardIndex = -1
+}
+
+func (shard *Shard) IsServicing() bool {
+ for _, slotRange := range shard.SlotRanges {
+ if slotRange.Start != -1 || slotRange.Stop != -1 {
+ return true
+ }
+ }
+ return shard.IsMigrating()
+}
+
+func (shard *Shard) addNode(addr, role, password string) error {
+ if role != RoleMaster && role != RoleSlave {
+ return fmt.Errorf("%w: role", consts.ErrInvalidArgument)
+ }
+ for _, node := range shard.Nodes {
+ if node.Addr() == addr {
+ return consts.ErrAlreadyExists
+ }
+ }
+ if role == RoleMaster && len(shard.Nodes) > 0 {
+ return fmt.Errorf("master node %w", consts.ErrAlreadyExists)
+ }
+ node := NewClusterNode(addr, password)
+ node.SetRole(role)
+ shard.Nodes = append(shard.Nodes, node)
+ return nil
+}
+
+func (shard *Shard) IsMigrating() bool {
+ return shard.MigratingSlot != -1 && shard.TargetShardIndex != -1
+}
+
+func (shard *Shard) GetMasterNode() Node {
+ for _, node := range shard.Nodes {
+ if node.IsMaster() {
+ return node
+ }
+ }
+ return nil
+}
+
+func (shard *Shard) removeNode(nodeID string) error {
+ isFound := false
+ for i, node := range shard.Nodes {
+ if node.ID() != nodeID {
+ continue
+ }
+ if node.IsMaster() {
+ return fmt.Errorf("cannot remove master node: %w", consts.ErrInvalidArgument)
+ }
+ shard.Nodes = append(shard.Nodes[:i], shard.Nodes[i+1:]...)
+ isFound = true
+ }
+ if !isFound {
+ return consts.ErrNotFound
+ }
+ return nil
+}
+
+func (shard *Shard) getNewMasterNodeIndex(ctx context.Context, masterNodeIndex int, preferredNodeID string) int {
+ newMasterNodeIndex := -1
+ var newestOffset uint64
+ for i, node := range shard.Nodes {
+ // don't promote the current master node
+ if i == masterNodeIndex {
+ continue
+ }
+ clusterNodeInfo, err := node.GetClusterNodeInfo(ctx)
+ if err != nil {
+ logger.Get().With(
+ zap.Error(err),
+ zap.String("id", node.ID()),
+ zap.String("addr", node.Addr()),
+ ).Warn("Skip the node due to failed to get cluster info")
+ continue
+ }
+ // If the preferredNodeID is not empty, we will use it as the new master node.
+ if preferredNodeID != "" && node.ID() == preferredNodeID {
+ newMasterNodeIndex = i
+ break
+ }
+ if clusterNodeInfo.Sequence >= newestOffset {
+ newMasterNodeIndex = i
+ newestOffset = clusterNodeInfo.Sequence
+ }
+ }
+ return newMasterNodeIndex
+}
+
+// PromoteNewMaster promotes a new master node in the shard,
+// it will return the new master node ID.
+//
+// The masterNodeID is used to check if the node is the current master node if it's not empty.
+// The preferredNodeID is used to specify the preferred node to be promoted as the new master node,
+// it will choose the node with the highest sequence number if the preferredNodeID is empty.
+func (shard *Shard) promoteNewMaster(ctx context.Context, masterNodeID, preferredNodeID string) (string, error) {
+ if len(shard.Nodes) <= 1 {
+ return "", consts.ErrShardNoReplica
+ }
+
+ oldMasterNodeIndex := -1
+ for i, node := range shard.Nodes {
+ if node.IsMaster() {
+ oldMasterNodeIndex = i
+ break
+ }
+ }
+ if oldMasterNodeIndex == -1 {
+ return "", consts.ErrOldMasterNodeNotFound
+ }
+ if masterNodeID != "" && shard.Nodes[oldMasterNodeIndex].ID() != masterNodeID {
+ return "", consts.ErrNodeIsNotMaster
+ }
+ newMasterNodeIndex := shard.getNewMasterNodeIndex(ctx, oldMasterNodeIndex, preferredNodeID)
+ if newMasterNodeIndex == -1 {
+ return "", consts.ErrShardNoMatchNewMaster
+ }
+ shard.Nodes[oldMasterNodeIndex].SetRole(RoleSlave)
+ shard.Nodes[newMasterNodeIndex].SetRole(RoleMaster)
+ preferredNewMasterNode := shard.Nodes[newMasterNodeIndex]
+ return preferredNewMasterNode.ID(), nil
+}
+
+func (shard *Shard) HasOverlap(slotRange *SlotRange) bool {
+ for _, shardSlotRange := range shard.SlotRanges {
+ if shardSlotRange.HasOverlap(slotRange) {
+ return true
+ }
+ }
+ return false
+}
+
+func (shard *Shard) ToSlotsString() (string, error) {
+ var builder strings.Builder
+ masterNodeIndex := -1
+ for i, node := range shard.Nodes {
+ if node.IsMaster() {
+ masterNodeIndex = i
+ break
+ }
+ }
+ if masterNodeIndex == -1 {
+ return "", errors.New("missing master node")
+ }
+
+ for i, node := range shard.Nodes {
+ builder.WriteString(node.ID())
+ builder.WriteByte(' ')
+ builder.WriteString(strings.Replace(node.Addr(), ":", " ", 1))
+ builder.WriteByte(' ')
+ if i == masterNodeIndex {
+ builder.WriteString(RoleMaster)
+ builder.WriteByte(' ')
+ builder.WriteByte('-')
+ builder.WriteByte(' ')
+ for j, slotRange := range shard.SlotRanges {
+ builder.WriteString(slotRange.String())
+ if j != len(shard.SlotRanges)-1 {
+ builder.WriteByte(' ')
+ }
+ }
+ } else {
+ builder.WriteString(RoleSlave)
+ builder.WriteByte(' ')
+ builder.WriteString(shard.Nodes[masterNodeIndex].ID())
+ }
+ builder.WriteByte('\n')
+ }
+ return builder.String(), nil
+}
+
+// UnmarshalJSON unmarshal a Shard from JSON bytes,
+// it's required since Shard.Nodes is an interface slice.
+// So we need to take into a concrete type.
+func (shard *Shard) UnmarshalJSON(bytes []byte) error {
+ var data struct {
+ SlotRanges []SlotRange `json:"slot_ranges"`
+ TargetShardIndex int `json:"target_shard_index"`
+ MigratingSlot int `json:"migrating_slot"`
+ Nodes []*ClusterNode `json:"nodes"`
+ }
+ if err := json.Unmarshal(bytes, &data); err != nil {
+ return err
+ }
+ shard.SlotRanges = data.SlotRanges
+ shard.TargetShardIndex = data.TargetShardIndex
+ shard.MigratingSlot = data.MigratingSlot
+ shard.Nodes = make([]Node, len(data.Nodes))
+ for i, node := range data.Nodes {
+ shard.Nodes[i] = node
+ }
+ return nil
+}
diff --git a/store/cluster_shard_test.go b/store/cluster_shard_test.go
new file mode 100644
index 0000000..b65cae2
--- /dev/null
+++ b/store/cluster_shard_test.go
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package store
+
+import (
+ "sort"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestShard_HasOverlap(t *testing.T) {
+ shard := NewShard()
+ slotRange := &SlotRange{Start: 0, Stop: 100}
+ shard.SlotRanges = append(shard.SlotRanges, *slotRange)
+ require.True(t, shard.HasOverlap(slotRange))
+ require.True(t, shard.HasOverlap(&SlotRange{Start: 50, Stop: 150}))
+ require.False(t, shard.HasOverlap(&SlotRange{Start: 101, Stop: 150}))
+}
+
+func TestShard_Sort(t *testing.T) {
+ shard0 := NewShard()
+ shard0.SlotRanges = []SlotRange{{Start: 201, Stop: 300}}
+ shard1 := NewShard()
+ shard1.SlotRanges = []SlotRange{{Start: 0, Stop: 400}}
+ shard2 := NewShard()
+ shard2.SlotRanges = []SlotRange{{Start: 101, Stop: 500}}
+ shards := Shards{shard0, shard1, shard2}
+ sort.Sort(shards)
+ require.EqualValues(t, 0, shards[0].SlotRanges[0].Start)
+ require.EqualValues(t, 101, shards[1].SlotRanges[0].Start)
+ require.EqualValues(t, 201, shards[2].SlotRanges[0].Start)
+}
+
+func TestShard_IsServicing(t *testing.T) {
+ shard := NewShard()
+ shard.TargetShardIndex = 0
+ shard.MigratingSlot = -1
+ require.False(t, shard.IsServicing())
+
+ shard.TargetShardIndex = 0
+ shard.MigratingSlot = 0
+ require.True(t, shard.IsServicing())
+
+ shard.TargetShardIndex = -1
+ shard.MigratingSlot = -1
+ shard.SlotRanges = []SlotRange{{Start: 0, Stop: 100}}
+ require.True(t, shard.IsServicing())
+
+ shard.SlotRanges = []SlotRange{{Start: -1, Stop: -1}}
+ require.False(t, shard.IsServicing())
+}
diff --git a/store/cluster_test.go b/store/cluster_test.go
new file mode 100644
index 0000000..fbfe34c
--- /dev/null
+++ b/store/cluster_test.go
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package store
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/kvrocks-controller/consts"
+)
+
+func TestCluster_PromoteNewMaster(t *testing.T) {
+ shard := NewShard()
+ shard.SlotRanges = []SlotRange{{Start: 0, Stop: 1023}}
+
+ node0 := NewClusterMockNode()
+ node0.SetRole(RoleMaster)
+
+ node1 := NewClusterMockNode()
+ node1.SetRole(RoleSlave)
+ node1.Sequence = 200
+
+ node2 := NewClusterMockNode()
+ node2.SetRole(RoleSlave)
+ node2.Sequence = 100
+
+ node3 := NewClusterMockNode()
+ node3.SetRole(RoleSlave)
+ node3.Sequence = 300
+
+ shard.Nodes = []Node{node0}
+ cluster := &Cluster{
+ Shards: Shards{shard},
+ }
+
+ ctx := context.Background()
+ _, err := cluster.PromoteNewMaster(ctx, -1, node0.ID(), "")
+ require.ErrorIs(t, err, consts.ErrIndexOutOfRange)
+ _, err = cluster.PromoteNewMaster(ctx, 1, node0.ID(), "")
+ require.ErrorIs(t, err, consts.ErrIndexOutOfRange)
+ _, err = cluster.PromoteNewMaster(ctx, 0, node0.ID(), "")
+ require.ErrorIs(t, err, consts.ErrShardNoReplica)
+
+ shard.Nodes = append(shard.Nodes, node1, node2, node3)
+ _, err = cluster.PromoteNewMaster(ctx, 0, node1.ID(), "")
+ require.ErrorIs(t, err, consts.ErrNodeIsNotMaster)
+
+ newMasterID, err := cluster.PromoteNewMaster(ctx, 0, node0.ID(), "")
+ require.NoError(t, err)
+ require.Equal(t, node3.ID(), newMasterID)
+
+ // test preferredNodeID
+ newMasterID, err = cluster.PromoteNewMaster(ctx, 0, node3.ID(), node2.ID())
+ require.NoError(t, err)
+ require.Equal(t, node2.ID(), newMasterID)
+}
diff --git a/store/engine/engine.go b/store/engine/engine.go
new file mode 100644
index 0000000..5052bd0
--- /dev/null
+++ b/store/engine/engine.go
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package engine
+
+import (
+ "context"
+ "strings"
+ "sync"
+
+ "github.com/apache/kvrocks-controller/consts"
+)
+
+type Entry struct {
+ Key string
+ Value []byte
+}
+
+type Engine interface {
+ ID() string
+ Leader() string
+ LeaderChange() <-chan bool
+ IsReady(ctx context.Context) bool
+
+ Get(ctx context.Context, key string) ([]byte, error)
+ Exists(ctx context.Context, key string) (bool, error)
+ Set(ctx context.Context, key string, value []byte) error
+ Delete(ctx context.Context, key string) error
+ List(ctx context.Context, prefix string) ([]Entry, error)
+
+ Close() error
+}
+
+type Mock struct {
+ mu sync.Mutex
+ values map[string]string
+}
+
+func NewMock() *Mock {
+ return &Mock{
+ values: make(map[string]string),
+ }
+}
+
+func (m *Mock) Get(ctx context.Context, key string) ([]byte, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ v, ok := m.values[key]
+ if !ok {
+ return nil, consts.ErrNotFound
+ }
+ return []byte(v), nil
+}
+
+func (m *Mock) Exists(ctx context.Context, key string) (bool, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ _, ok := m.values[key]
+ return ok, nil
+}
+
+func (m *Mock) Set(ctx context.Context, key string, value []byte) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.values[key] = string(value)
+ return nil
+}
+
+func (m *Mock) Delete(ctx context.Context, key string) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ delete(m.values, key)
+ return nil
+}
+
+func (m *Mock) List(ctx context.Context, prefix string) ([]Entry, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ var entries []Entry
+ for k, v := range m.values {
+ if k == prefix || (len(k) > len(prefix) && k[len(prefix)] == '/') {
+ entries = append(entries, Entry{
+ Key: strings.TrimPrefix(k, prefix+"/"),
+ Value: []byte(v),
+ })
+ }
+ }
+ return entries, nil
+}
+
+func (m *Mock) Close() error {
+ return nil
+}
+
+func (m *Mock) ID() string {
+ return "mock_store_engine"
+}
+
+func (m *Mock) Leader() string {
+ return "mock_store_engine"
+}
+
+func (m *Mock) LeaderChange() <-chan bool {
+ return make(chan bool)
+}
+
+func (m *Mock) IsReady(ctx context.Context) bool {
+ return true
+}
+
+var _ Engine = (*Mock)(nil)
diff --git a/storage/persistence/etcd/etcd.go b/store/engine/etcd/etcd.go
similarity index 94%
rename from storage/persistence/etcd/etcd.go
rename to store/engine/etcd/etcd.go
index 399e831..4920cfa 100644
--- a/storage/persistence/etcd/etcd.go
+++ b/store/engine/etcd/etcd.go
@@ -22,6 +22,7 @@ package etcd
import (
"context"
"errors"
+ "github.com/apache/kvrocks-controller/consts"
"strings"
"sync"
"time"
@@ -33,8 +34,7 @@ import (
"go.uber.org/zap"
"github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/apache/kvrocks-controller/storage/persistence"
+ "github.com/apache/kvrocks-controller/store/engine"
)
const (
@@ -162,7 +162,7 @@ func (e *Etcd) Get(ctx context.Context, key string) ([]byte, error) {
return nil, err
}
if len(rsp.Kvs) == 0 {
- return nil, metadata.ErrEntryNoExists
+ return nil, consts.ErrNotFound
}
return rsp.Kvs[0].Value, nil
}
@@ -170,7 +170,7 @@ func (e *Etcd) Get(ctx context.Context, key string) ([]byte, error) {
func (e *Etcd) Exists(ctx context.Context, key string) (bool, error) {
_, err := e.Get(ctx, key)
if err != nil {
- if errors.Is(err, metadata.ErrEntryNoExists) {
+ if errors.Is(err, consts.ErrNotFound) {
return false, nil
}
return false, err
@@ -188,14 +188,14 @@ func (e *Etcd) Delete(ctx context.Context, key string) error {
return err
}
-func (e *Etcd) List(ctx context.Context, prefix string) ([]persistence.Entry, error) {
+func (e *Etcd) List(ctx context.Context, prefix string) ([]engine.Entry, error) {
rsp, err := e.kv.Get(ctx, prefix, clientv3.WithPrefix())
if err != nil {
return nil, err
}
prefixLen := len(prefix)
- entries := make([]persistence.Entry, 0)
+ entries := make([]engine.Entry, 0)
for _, kv := range rsp.Kvs {
if string(kv.Key) == prefix {
continue
@@ -204,7 +204,7 @@ func (e *Etcd) List(ctx context.Context, prefix string) ([]persistence.Entry, er
if strings.ContainsRune(key, '/') {
continue
}
- entries = append(entries, persistence.Entry{
+ entries = append(entries, engine.Entry{
Key: key,
Value: kv.Value,
})
diff --git a/storage/persistence/etcd/etcd_test.go b/store/engine/etcd/etcd_test.go
similarity index 100%
rename from storage/persistence/etcd/etcd_test.go
rename to store/engine/etcd/etcd_test.go
diff --git a/storage/persistence/zookeeper/zookeeper.go b/store/engine/zookeeper/zookeeper.go
similarity index 96%
rename from storage/persistence/zookeeper/zookeeper.go
rename to store/engine/zookeeper/zookeeper.go
index 5b00a6e..7e863a3 100644
--- a/storage/persistence/zookeeper/zookeeper.go
+++ b/store/engine/zookeeper/zookeeper.go
@@ -27,7 +27,7 @@ import (
"time"
"github.com/apache/kvrocks-controller/logger"
- "github.com/apache/kvrocks-controller/storage/persistence"
+ "github.com/apache/kvrocks-controller/store/engine"
"github.com/go-zookeeper/zk"
"go.uber.org/atomic"
)
@@ -179,15 +179,15 @@ func (e *Zookeeper) Delete(ctx context.Context, key string) error {
return err
}
-func (e *Zookeeper) List(ctx context.Context, prefix string) ([]persistence.Entry, error) {
+func (e *Zookeeper) List(ctx context.Context, prefix string) ([]engine.Entry, error) {
children, _, err := e.conn.Children(prefix)
if errors.Is(err, zk.ErrNoNode) {
- return []persistence.Entry{}, nil
+ return []engine.Entry{}, nil
} else if err != nil {
return nil, err
}
- entries := make([]persistence.Entry, 0)
+ entries := make([]engine.Entry, 0)
for _, child := range children {
key := prefix + "/" + child
data, _, err := e.conn.Get(key)
@@ -195,7 +195,7 @@ func (e *Zookeeper) List(ctx context.Context, prefix string) ([]persistence.Entr
return nil, err
}
- entry := persistence.Entry{
+ entry := engine.Entry{
Key: child,
Value: data,
}
diff --git a/storage/persistence/zookeeper/zookeeper_test.go b/store/engine/zookeeper/zookeeper_test.go
similarity index 100%
rename from storage/persistence/zookeeper/zookeeper_test.go
rename to store/engine/zookeeper/zookeeper_test.go
diff --git a/storage/event.go b/store/event.go
similarity index 84%
rename from storage/event.go
rename to store/event.go
index 1accbd7..2a813dd 100644
--- a/storage/event.go
+++ b/store/event.go
@@ -17,7 +17,7 @@
* under the License.
*
*/
-package storage
+package store
type EventType int
type Command int
@@ -25,25 +25,17 @@ type Command int
const (
EventNamespace EventType = iota + 1
EventCluster
- EventShard
- EventNode
)
const (
CommandCreate = iota + 1
+ CommandUpdate = iota + 1
CommandRemove
- CommandUpdate
- CommandAddSlots
- CommandRemoveSlots
- CommandMigrateSlots
)
-type Event struct {
+type EventPayload struct {
Namespace string
Cluster string
- Shard int
- NodeID string
Type EventType
Command Command
- Data interface{}
}
diff --git a/cmd/client/error.go b/store/helper.go
similarity index 70%
rename from cmd/client/error.go
rename to store/helper.go
index 92ca757..5bb4f65 100644
--- a/cmd/client/error.go
+++ b/store/helper.go
@@ -17,13 +17,22 @@
* under the License.
*
*/
-package main
+package store
-import "errors"
-
-var (
- ErrUnknownCommand = errors.New("unknown command")
- ErrWrongArguments = errors.New("wrong arguments")
- ErrNamespaceNotExits = errors.New("namespace does not exist")
- ErrClusterNotExits = errors.New("cluster does not exist")
+import (
+ "fmt"
)
+
+const nsPrefix = "/kvrocks/metadata"
+
+func appendPrefix(ns string) string {
+ return nsPrefix + "/" + ns
+}
+
+func buildClusterPrefix(ns string) string {
+ return fmt.Sprintf("%s/%s/cluster", nsPrefix, ns)
+}
+
+func buildClusterKey(ns, cluster string) string {
+ return fmt.Sprintf("%s/%s", buildClusterPrefix(ns), cluster)
+}
diff --git a/metadata/slot.go b/store/slot.go
similarity index 97%
rename from metadata/slot.go
rename to store/slot.go
index 3b540dc..db088db 100644
--- a/metadata/slot.go
+++ b/store/slot.go
@@ -17,7 +17,7 @@
* under the License.
*
*/
-package metadata
+package store
import (
"encoding/json"
@@ -57,6 +57,10 @@ func (slotRange *SlotRange) HasOverlap(that *SlotRange) bool {
return !(slotRange.Stop < that.Start || slotRange.Start > that.Stop)
}
+func (slotRange *SlotRange) Contains(slot int) bool {
+ return slot >= slotRange.Start && slot <= slotRange.Stop
+}
+
func (slotRange *SlotRange) String() string {
if slotRange.Start == slotRange.Stop {
return strconv.Itoa(slotRange.Start)
diff --git a/metadata/slot_test.go b/store/slot_test.go
similarity index 99%
rename from metadata/slot_test.go
rename to store/slot_test.go
index 1841257..e5d2ce2 100644
--- a/metadata/slot_test.go
+++ b/store/slot_test.go
@@ -17,7 +17,7 @@
* under the License.
*
*/
-package metadata
+package store
import (
"testing"
diff --git a/store/store.go b/store/store.go
new file mode 100644
index 0000000..84dea95
--- /dev/null
+++ b/store/store.go
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+type Store interface {
+ IsReady(ctx context.Context) bool
+
+ ListNamespace(ctx context.Context) ([]string, error)
+ CreateNamespace(ctx context.Context, ns string) error
+ ExistsNamespace(ctx context.Context, ns string) (bool, error)
+ RemoveNamespace(ctx context.Context, ns string) error
+
+ ListCluster(ctx context.Context, ns string) ([]string, error)
+ GetCluster(ctx context.Context, ns, cluster string) (*Cluster, error)
+ RemoveCluster(ctx context.Context, ns, cluster string) error
+ CreateCluster(ctx context.Context, ns string, cluster *Cluster) error
+ UpdateCluster(ctx context.Context, ns string, cluster *Cluster) error
+ SetCluster(ctx context.Context, ns string, clusterInfo *Cluster) error
+}
+
+var _ Store = (*ClusterStore)(nil)
+
+type ClusterStore struct {
+ e engine.Engine
+
+ eventNotifyCh chan EventPayload
+ quitCh chan struct{}
+}
+
+func NewClusterStore(e engine.Engine) *ClusterStore {
+ return &ClusterStore{
+ e: e,
+ eventNotifyCh: make(chan EventPayload, 100),
+ quitCh: make(chan struct{}),
+ }
+}
+
+func (s *ClusterStore) IsReady(ctx context.Context) bool {
+ return s.e.IsReady(ctx)
+}
+
+// ListNamespace return the list of name of all namespaces
+func (s *ClusterStore) ListNamespace(ctx context.Context) ([]string, error) {
+ entries, err := s.e.List(ctx, nsPrefix)
+ if err != nil {
+ return nil, err
+ }
+ keys := make([]string, len(entries))
+ for i, entry := range entries {
+ keys[i] = entry.Key
+ }
+ return keys, nil
+}
+
+// ExistsNamespace return an indicator whether the specified namespace exists
+func (s *ClusterStore) ExistsNamespace(ctx context.Context, ns string) (bool, error) {
+ return s.e.Exists(ctx, appendPrefix(ns))
+}
+
+// CreateNamespace will create a namespace for clusters
+func (s *ClusterStore) CreateNamespace(ctx context.Context, ns string) error {
+ if has, _ := s.ExistsNamespace(ctx, ns); has {
+ return consts.ErrAlreadyExists
+ }
+ if err := s.e.Set(ctx, appendPrefix(ns), []byte(ns)); err != nil {
+ return err
+ }
+ s.EmitEvent(EventPayload{
+ Namespace: ns,
+ Type: EventNamespace,
+ Command: CommandCreate,
+ })
+ return nil
+}
+
+// RemoveNamespace delete the specified namespace from store
+func (s *ClusterStore) RemoveNamespace(ctx context.Context, ns string) error {
+ if has, _ := s.ExistsNamespace(ctx, ns); !has {
+ return consts.ErrNotFound
+ }
+ clusters, err := s.ListCluster(ctx, ns)
+ if err != nil {
+ return err
+ }
+ if len(clusters) != 0 {
+ return errors.New("namespace wasn't empty, please remove clusters first")
+ }
+ if err := s.e.Delete(ctx, appendPrefix(ns)); err != nil {
+ return err
+ }
+ s.EmitEvent(EventPayload{
+ Namespace: ns,
+ Type: EventNamespace,
+ Command: CommandRemove,
+ })
+ return nil
+}
+
+// ListCluster return the list of name of cluster under the specified namespace
+func (s *ClusterStore) ListCluster(ctx context.Context, ns string) ([]string, error) {
+ entries, err := s.e.List(ctx, buildClusterPrefix(ns))
+ if err != nil {
+ return nil, err
+ }
+ keys := make([]string, len(entries))
+ for i, entry := range entries {
+ keys[i] = entry.Key
+ }
+ return keys, nil
+}
+
+func (s *ClusterStore) existsCluster(ctx context.Context, ns, cluster string) (bool, error) {
+ return s.e.Exists(ctx, buildClusterKey(ns, cluster))
+}
+
+func (s *ClusterStore) GetCluster(ctx context.Context, ns, cluster string) (*Cluster, error) {
+ value, err := s.e.Get(ctx, buildClusterKey(ns, cluster))
+ if err != nil {
+ return nil, err
+ }
+ var clusterInfo Cluster
+ if err = json.Unmarshal(value, &clusterInfo); err != nil {
+ return nil, err
+ }
+ return &clusterInfo, nil
+}
+
+// UpdateCluster update the Name to store under the specified namespace
+func (s *ClusterStore) UpdateCluster(ctx context.Context, ns string, clusterInfo *Cluster) error {
+ clusterInfo.Version.Inc()
+ if err := s.SetCluster(ctx, ns, clusterInfo); err != nil {
+ return err
+ }
+ s.EmitEvent(EventPayload{
+ Namespace: ns,
+ Cluster: clusterInfo.Name,
+ Type: EventCluster,
+ Command: CommandUpdate,
+ })
+ return nil
+}
+
+func (s *ClusterStore) SetCluster(ctx context.Context, ns string, clusterInfo *Cluster) error {
+ if len(clusterInfo.Shards) == 0 {
+ return errors.New("required at least one shard")
+ }
+ value, err := json.Marshal(clusterInfo)
+ if err != nil {
+ return err
+ }
+ return s.e.Set(ctx, buildClusterKey(ns, clusterInfo.Name), value)
+}
+
+func (s *ClusterStore) CreateCluster(ctx context.Context, ns string, clusterInfo *Cluster) error {
+ if exists, _ := s.existsCluster(ctx, ns, clusterInfo.Name); exists {
+ return consts.ErrAlreadyExists
+ }
+ if err := s.SetCluster(ctx, ns, clusterInfo); err != nil {
+ return err
+ }
+ s.EmitEvent(EventPayload{
+ Namespace: ns,
+ Cluster: clusterInfo.Name,
+ Type: EventCluster,
+ Command: CommandCreate,
+ })
+ return nil
+}
+
+func (s *ClusterStore) RemoveCluster(ctx context.Context, ns, cluster string) error {
+ if exists, _ := s.existsCluster(ctx, ns, cluster); !exists {
+ return consts.ErrNotFound
+ }
+ if err := s.e.Delete(ctx, buildClusterKey(ns, cluster)); err != nil {
+ return err
+ }
+ s.EmitEvent(EventPayload{
+ Namespace: ns,
+ Cluster: cluster,
+ Type: EventCluster,
+ Command: CommandRemove,
+ })
+ return nil
+}
+
+func (s *ClusterStore) Notify() <-chan EventPayload {
+ return s.eventNotifyCh
+}
+
+func (s *ClusterStore) EmitEvent(event EventPayload) {
+ s.eventNotifyCh <- event
+}
+
+func (s *ClusterStore) LeaderChange() <-chan bool {
+ return s.e.LeaderChange()
+}
+
+func (s *ClusterStore) IsLeader() bool {
+ return s.e.Leader() == s.e.ID()
+}
+
+func (s *ClusterStore) Leader() string {
+ return s.e.Leader()
+}
+
+func (s *ClusterStore) ID() string {
+ return s.e.ID()
+}
+
+func (s *ClusterStore) Close() error {
+ return s.e.Close()
+}
+
+func (s *ClusterStore) Stop() error {
+ return nil
+}
diff --git a/store/store_test.go b/store/store_test.go
new file mode 100644
index 0000000..496a013
--- /dev/null
+++ b/store/store_test.go
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package store
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/apache/kvrocks-controller/consts"
+ "github.com/apache/kvrocks-controller/store/engine"
+)
+
+func TestClusterStore(t *testing.T) {
+ ctx := context.Background()
+ store := NewClusterStore(engine.NewMock())
+
+ t.Run("create/get/list/delete namespace", func(t *testing.T) {
+ namespaces := []string{"ns0", "ns1", "ns2"}
+ for _, ns := range namespaces {
+ err := store.CreateNamespace(ctx, ns)
+ require.NoError(t, err)
+ require.ErrorIs(t, store.CreateNamespace(ctx, ns), consts.ErrAlreadyExists)
+ exists, err := store.ExistsNamespace(ctx, ns)
+ require.NoError(t, err)
+ require.True(t, exists)
+ }
+
+ exists, err := store.ExistsNamespace(ctx, "not-exits-ns")
+ require.NoError(t, err)
+ require.False(t, exists)
+
+ gotNamespaces, err := store.ListNamespace(ctx)
+ require.NoError(t, err)
+ require.ElementsMatch(t, namespaces, gotNamespaces)
+
+ for _, ns := range namespaces {
+ err := store.RemoveNamespace(ctx, ns)
+ require.NoError(t, err)
+ exists, err := store.ExistsNamespace(ctx, ns)
+ require.NoError(t, err)
+ require.False(t, exists)
+ }
+ })
+
+ t.Run("create/get/list/delete cluster", func(t *testing.T) {
+ ns := "ns0"
+ cluster0 := &Cluster{Name: "cluster0", Shards: Shards{NewShard()}}
+ cluster1 := &Cluster{Name: "cluster1", Shards: Shards{NewShard()}}
+ cluster0.Version.Store(2)
+ cluster1.Version.Store(3)
+
+ require.NoError(t, store.CreateCluster(ctx, ns, cluster0))
+ require.ErrorIs(t, store.CreateCluster(ctx, ns, cluster0), consts.ErrAlreadyExists)
+ require.NoError(t, store.CreateCluster(ctx, ns, cluster1))
+
+ gotCluster, err := store.GetCluster(ctx, ns, "cluster0")
+ require.NoError(t, err)
+ require.Equal(t, cluster0.Name, gotCluster.Name)
+ require.Equal(t, cluster0.Version, gotCluster.Version)
+
+ gotClusters, err := store.ListCluster(ctx, ns)
+ require.NoError(t, err)
+ require.ElementsMatch(t, []string{"cluster0", "cluster1"}, gotClusters)
+
+ cluster0.Version.Store(4)
+ require.NoError(t, store.UpdateCluster(ctx, ns, cluster0))
+ gotCluster, err = store.GetCluster(ctx, ns, "cluster0")
+ require.NoError(t, err)
+ require.Equal(t, cluster0.Name, gotCluster.Name)
+ require.Equal(t, cluster0.Version, gotCluster.Version)
+
+ for _, name := range []string{"cluster0", "cluster1"} {
+ require.NoError(t, store.RemoveCluster(ctx, ns, name))
+ _, err = store.GetCluster(ctx, ns, name)
+ require.ErrorIs(t, err, consts.ErrNotFound)
+ }
+ })
+}
diff --git a/util/redis.go b/util/redis.go
deleted file mode 100644
index 12045a0..0000000
--- a/util/redis.go
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package util
-
-import (
- "context"
- "fmt"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/apache/kvrocks-controller/metadata"
- "github.com/go-redis/redis/v8"
-)
-
-var (
- pool map[string]*redis.Client
- poolMu sync.RWMutex
-)
-
-const (
- dialTimeout = 3200 * time.Millisecond
- readTimeout = 3 * time.Second
- writeTimeout = 3 * time.Second
- minIdleConns = 3
-)
-
-func init() {
- pool = make(map[string]*redis.Client)
-}
-
-func GetRedisClient(ctx context.Context, node *metadata.NodeInfo) (*redis.Client, error) {
- poolMu.RLock()
- if c, ok := pool[node.Addr]; ok {
- poolMu.RUnlock()
- return c, nil
- }
- poolMu.RUnlock()
-
- poolMu.Lock()
- defer poolMu.Unlock()
- client := redis.NewClient(&redis.Options{
- Addr: node.Addr,
- Password: node.Password,
- DialTimeout: dialTimeout,
- ReadTimeout: readTimeout,
- WriteTimeout: writeTimeout,
- MaxRetries: -1, // don't retry inside the client
- MinIdleConns: minIdleConns,
- })
- if err := client.Do(ctx, "ping").Err(); err != nil {
- return nil, err
- }
- pool[node.Addr] = client
- return client, nil
-}
-
-func CloseRedisClients() {
- poolMu.Lock()
- defer poolMu.Unlock()
- for addr, c := range pool {
- c.Close()
- delete(pool, addr)
- }
-}
-
-type ClusterInfo struct {
- ClusterState bool
- ClusterSlotsAssigned int
- ClusterSlotsOK int
- ClusterSlotsPFail int
- ClusterSlotsFail int
- ClusterKnownNodes int
- ClusterSize int
- ClusterCurrentEpoch int64
- ClusterMyEpoch int64
- MigratingSlot int
- ImportingSlot int
- DestinationNode string
- MigratingState string
- ImportingState string
-}
-
-func ClusterInfoCmd(ctx context.Context, node *metadata.NodeInfo) (*ClusterInfo, error) {
- cli, err := GetRedisClient(ctx, node)
- if err != nil {
- return nil, err
- }
- rsp, err := cli.ClusterInfo(ctx).Result()
- if err != nil {
- return nil, err
- }
- clusterInfo := &ClusterInfo{
- MigratingSlot: -1,
- ImportingSlot: -1,
- DestinationNode: "",
- MigratingState: "",
- ImportingState: "",
- }
- lines := strings.Split(rsp, "\n")
- for _, line := range lines {
- kv := strings.Split(line, ":")
- if len(kv) != 2 {
- continue
- }
- key := strings.TrimSpace(kv[0])
- val := strings.TrimSpace(kv[1])
- switch key {
- case "cluster_state":
- if val == "ok" {
- clusterInfo.ClusterState = true
- }
- case "cluster_slots_assigned":
- clusterInfo.ClusterSlotsAssigned, _ = strconv.Atoi(val)
- case "cluster_slots_ok":
- clusterInfo.ClusterSlotsOK, _ = strconv.Atoi(val)
- case "cluster_slots_pfail":
- clusterInfo.ClusterSlotsPFail, _ = strconv.Atoi(val)
- case "cluster_slots_fail":
- clusterInfo.ClusterSlotsFail, _ = strconv.Atoi(val)
- case "cluster_known_nodes":
- clusterInfo.ClusterKnownNodes, _ = strconv.Atoi(val)
- case "cluster_size":
- clusterInfo.ClusterSize, _ = strconv.Atoi(val)
- case "cluster_current_epoch":
- clusterInfo.ClusterCurrentEpoch, _ = strconv.ParseInt(val, 10, 64)
- case "cluster_my_epoch":
- clusterInfo.ClusterMyEpoch, _ = strconv.ParseInt(val, 10, 64)
- case "migrating_slot":
- clusterInfo.MigratingSlot, _ = strconv.Atoi(val)
- case "importing_slot":
- clusterInfo.ImportingSlot, _ = strconv.Atoi(val)
- case "destination_node":
- clusterInfo.DestinationNode = val
- case "migrating_state":
- clusterInfo.MigratingState = val
- case "import_state":
- clusterInfo.ImportingState = val
- }
- }
- return clusterInfo, nil
-}
-
-type ServerInfo struct {
- Version string
- GitSha1 string
- OS string
- GccVersion string
- ArchBits string
- ProcessId string
- TcpPort string
- UpTime string
-}
-
-type ClientInfo struct {
- MaxClients string
- ConnectedClients string
- MonitorClients string
-}
-
-type MemoryInfo struct {
- UsedMemoryRss string
- UsedMemoryHuman string
- UsedMemoryLua string
- UsedMemoryLuaHuman string
-}
-
-type StatesInfo struct {
- TotalConnectionsReceived string
- TotalCommandsProcessed string
- InstantaneousOps string
- TotalNetInputBytes string
- TotalNetOutputBytes string
- InstantaneousInputKbps string
- InstantaneousOutputKbps string
- SyncFull string
- SyncPartialOk string
- SyncPartialErr string
- PubsubChannels string
- PubsubPatterns string
-}
-
-type SlaveReplicationInfo struct {
- Role string
- MasterHost string
- MasterPort string
- MasterLinkStatus string
- MasterSyncUnrecoverableError string
- MasterSyncInProgress string
- MasterLastIoSecondsAgo string
- SlaveReplOffset string
- SlavePriority string
-}
-
-type MasterReplicationInfo struct {
- Role string
- ConnectedSlaves string
- MasterReplOffset string
-}
-
-type KeySpaceInfo struct {
- Sequence string
- UsedDbSize string
- MaxDbSize string
- UsedPercent string
- DiskCapacity string
- UsedDiskSize string
- UsedDiskPercent string
-}
... 184 lines suppressed ...