You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by tw...@apache.org on 2023/04/03 03:35:54 UTC
[incubator-kvrocks] branch unstable updated: Add handling LMOVE in BatchExtractor (#1364)
This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/incubator-kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 115f354b Add handling LMOVE in BatchExtractor (#1364)
115f354b is described below
commit 115f354b91cca2f647c04a8c1eebcfed6b9dd4a9
Author: Yaroslav <to...@gmail.com>
AuthorDate: Mon Apr 3 06:35:49 2023 +0300
Add handling LMOVE in BatchExtractor (#1364)
---
src/cluster/cluster.cc | 26 ++-
src/cluster/redis_slot.cc | 2 +-
src/cluster/redis_slot.h | 2 +-
src/commands/cmd_cluster.cc | 2 +-
src/storage/batch_extractor.cc | 187 +++++++++++++++------
src/storage/batch_extractor.h | 15 +-
src/storage/redis_db.cc | 2 +-
src/storage/redis_metadata.cc | 2 +-
src/types/redis_list.cc | 6 +-
.../integration/slotmigrate/slotmigrate_test.go | 69 ++++++++
10 files changed, 234 insertions(+), 79 deletions(-)
diff --git a/src/cluster/cluster.cc b/src/cluster/cluster.cc
index 2514e587..aa46a1f8 100644
--- a/src/cluster/cluster.cc
+++ b/src/cluster/cluster.cc
@@ -773,7 +773,7 @@ Status Cluster::CanExecByMySelf(const Redis::CommandAttributes *attributes, cons
for (auto i : keys_indexes) {
if (i >= static_cast<int>(cmd_tokens.size())) break;
- int cur_slot = GetSlotNumFromKey(cmd_tokens[i]);
+ int cur_slot = GetSlotIdFromKey(cmd_tokens[i]);
if (slot == -1) slot = cur_slot;
if (slot != cur_slot) {
return {Status::RedisExecErr, "CROSSSLOT Attempted to access keys that don't hash to the same slot"};
@@ -783,7 +783,9 @@ Status Cluster::CanExecByMySelf(const Redis::CommandAttributes *attributes, cons
if (slots_nodes_[slot] == nullptr) {
return {Status::ClusterDown, "CLUSTERDOWN Hash slot not served"};
- } else if (myself_ && myself_ == slots_nodes_[slot]) {
+ }
+
+ if (myself_ && myself_ == slots_nodes_[slot]) {
// We use central controller to manage the topology of the cluster.
// Server can't change the topology directly, so we record the migrated slots
// to move the requests of the migrated slots to the destination node.
@@ -797,22 +799,28 @@ Status Cluster::CanExecByMySelf(const Redis::CommandAttributes *attributes, cons
}
return Status::OK(); // I'm serving this slot
- } else if (myself_ && myself_->importing_slot_ == slot && conn->IsImporting()) {
+ }
+
+ if (myself_ && myself_->importing_slot_ == slot && conn->IsImporting()) {
// While data migrating, the topology of the destination node has not been changed.
// The destination node has to serve the requests from the migrating slot,
// although the slot is not belong to itself. Therefore, we record the importing slot
// and mark the importing connection to accept the importing data.
return Status::OK(); // I'm serving the importing connection
- } else if (myself_ && imported_slots_.count(slot)) {
+ }
+
+ if (myself_ && imported_slots_.count(slot)) {
// After the slot is migrated, new requests of the migrated slot will be moved to
// the destination server. Before the central controller change the topology, the destination
// server should record the imported slots to accept new data of the imported slots.
return Status::OK(); // I'm serving the imported slot
- } else if (myself_ && myself_->role_ == kClusterSlave && !attributes->is_write() &&
- nodes_.find(myself_->master_id_) != nodes_.end() && nodes_[myself_->master_id_] == slots_nodes_[slot]) {
+ }
+
+ if (myself_ && myself_->role_ == kClusterSlave && !attributes->is_write() &&
+ nodes_.find(myself_->master_id_) != nodes_.end() && nodes_[myself_->master_id_] == slots_nodes_[slot]) {
return Status::OK(); // My master is serving this slot
- } else {
- return {Status::RedisExecErr,
- fmt::format("MOVED {} {}:{}", slot, slots_nodes_[slot]->host_, slots_nodes_[slot]->port_)};
}
+
+ return {Status::RedisExecErr,
+ fmt::format("MOVED {} {}:{}", slot, slots_nodes_[slot]->host_, slots_nodes_[slot]->port_)};
}
diff --git a/src/cluster/redis_slot.cc b/src/cluster/redis_slot.cc
index 0a58d742..b5487d6e 100644
--- a/src/cluster/redis_slot.cc
+++ b/src/cluster/redis_slot.cc
@@ -54,7 +54,7 @@ uint16_t crc16(const char *buf, int len) {
return crc;
}
-uint16_t GetSlotNumFromKey(const std::string &key) {
+uint16_t GetSlotIdFromKey(const std::string &key) {
auto tag = GetTagFromKey(key);
if (tag.empty()) {
tag = key;
diff --git a/src/cluster/redis_slot.h b/src/cluster/redis_slot.h
index a940f6c1..4354597c 100644
--- a/src/cluster/redis_slot.h
+++ b/src/cluster/redis_slot.h
@@ -27,5 +27,5 @@ constexpr const uint16_t HASH_SLOTS_SIZE = HASH_SLOTS_MASK + 1; // 16384
constexpr const uint16_t HASH_SLOTS_MAX_ITERATIONS = 50;
uint16_t crc16(const char *buf, int len);
-uint16_t GetSlotNumFromKey(const std::string &key);
+uint16_t GetSlotIdFromKey(const std::string &key);
std::string GetTagFromKey(const std::string &key);
diff --git a/src/commands/cmd_cluster.cc b/src/commands/cmd_cluster.cc
index 72b3eac5..682c0c40 100644
--- a/src/commands/cmd_cluster.cc
+++ b/src/commands/cmd_cluster.cc
@@ -60,7 +60,7 @@ class CommandCluster : public Commander {
}
if (subcommand_ == "keyslot") {
- auto slot_id = GetSlotNumFromKey(args_[2]);
+ auto slot_id = GetSlotIdFromKey(args_[2]);
*output = Redis::Integer(slot_id);
} else if (subcommand_ == "slots") {
std::vector<SlotInfo> infos;
diff --git a/src/storage/batch_extractor.cc b/src/storage/batch_extractor.cc
index bb1826b1..17984309 100644
--- a/src/storage/batch_extractor.cc
+++ b/src/storage/batch_extractor.cc
@@ -21,7 +21,6 @@
#include "batch_extractor.h"
#include <glog/logging.h>
-#include <rocksdb/write_batch.h>
#include "cluster/redis_slot.h"
#include "parse_util.h"
@@ -49,15 +48,18 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
return rocksdb::Status::OK();
}
- std::string ns, user_key, sub_key;
+ std::string ns, user_key;
std::vector<std::string> command_args;
+
if (column_family_id == kColumnFamilyIDMetadata) {
- ExtractNamespaceKey(key, &ns, &user_key, is_slotid_encoded_);
- if (slot_ >= 0) {
- if (static_cast<uint16_t>(slot_) != GetSlotNumFromKey(user_key)) return rocksdb::Status::OK();
+ ExtractNamespaceKey(key, &ns, &user_key, is_slot_id_encoded_);
+ if (slot_id_ >= 0 && static_cast<uint16_t>(slot_id_) != GetSlotIdFromKey(user_key)) {
+ return rocksdb::Status::OK();
}
+
Metadata metadata(kRedisNone);
metadata.Decode(value.ToString());
+
if (metadata.Type() == kRedisString) {
command_args = {"SET", user_key, value.ToString().substr(Metadata::GetOffsetAfterExpire(value[0]))};
resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args));
@@ -70,8 +72,10 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
if (args->size() > 0) {
auto parse_result = ParseInt<int>((*args)[0], 10);
if (!parse_result) {
- return rocksdb::Status::InvalidArgument(parse_result.Msg());
+ return rocksdb::Status::InvalidArgument(
+ fmt::format("failed to parse Redis command from log data: {}", parse_result.Msg()));
}
+
auto cmd = static_cast<RedisCommand>(*parse_result);
if (cmd == kRedisCmdExpire) {
command_args = {"PEXPIREAT", user_key, std::to_string(metadata.expire)};
@@ -82,13 +86,15 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
if (metadata.Type() == kRedisStream) {
auto args = log_data_.GetArguments();
- bool isSetID = args && args->size() > 0 && (*args)[0] == "XSETID";
- if (!isSetID) {
+ bool is_set_id = args && args->size() > 0 && (*args)[0] == "XSETID";
+ if (!is_set_id) {
return rocksdb::Status::OK();
}
+
StreamMetadata stream_metadata;
auto s = stream_metadata.Decode(value.ToString());
if (!s.ok()) return s;
+
command_args = {"XSETID",
user_key,
stream_metadata.last_entry_id.ToString(),
@@ -98,54 +104,76 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
stream_metadata.max_deleted_entry_id.ToString()};
resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args));
}
+
return rocksdb::Status::OK();
- } else if (column_family_id == kColumnFamilyIDDefault) {
- InternalKey ikey(key, is_slotid_encoded_);
+ }
+
+ if (column_family_id == kColumnFamilyIDDefault) {
+ InternalKey ikey(key, is_slot_id_encoded_);
user_key = ikey.GetKey().ToString();
- if (slot_ >= 0) {
- if (static_cast<uint16_t>(slot_) != GetSlotNumFromKey(user_key)) return rocksdb::Status::OK();
+ if (slot_id_ >= 0 && static_cast<uint16_t>(slot_id_) != GetSlotIdFromKey(user_key)) {
+ return rocksdb::Status::OK();
}
- sub_key = ikey.GetSubKey().ToString();
+
+ std::string sub_key = ikey.GetSubKey().ToString();
ns = ikey.GetNamespace().ToString();
+
switch (log_data_.GetRedisType()) {
case kRedisHash:
command_args = {"HSET", user_key, sub_key, value.ToString()};
break;
case kRedisList: {
auto args = log_data_.GetArguments();
- if (args->size() < 1) {
- LOG(ERROR) << "Fail to parse write_batch in putcf type list : args error ,should at least contain cmd";
+ if (args->empty()) {
+ LOG(ERROR)
+ << "Failed to parse write_batch in PutCF. Type=List: no arguments, at least should contain a command";
return rocksdb::Status::OK();
}
+
auto parse_result = ParseInt<int>((*args)[0], 10);
if (!parse_result) {
- return rocksdb::Status::InvalidArgument(parse_result.Msg());
+ return rocksdb::Status::InvalidArgument(
+ fmt::format("failed to parse Redis command from log data: {}", parse_result.Msg()));
}
+
auto cmd = static_cast<RedisCommand>(*parse_result);
switch (cmd) {
case kRedisCmdLSet:
if (args->size() < 2) {
- LOG(ERROR) << "Fail to parse write_batch in putcf cmd lset : args error ,should contain lset index";
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Command=LSET: no enough arguments, at least should "
+ "contain an index";
return rocksdb::Status::OK();
}
+
command_args = {"LSET", user_key, (*args)[1], value.ToString()};
break;
case kRedisCmdLInsert:
if (first_seen_) {
if (args->size() < 4) {
- LOG(ERROR)
- << "Fail to parse write_batch in putcf cmd linsert : args error, should contain before pivot value";
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Command=LINSERT: no enough arguments, should "
+ "contain before pivot value";
return rocksdb::Status::OK();
}
+
command_args = {"LINSERT", user_key, (*args)[1] == "1" ? "before" : "after", (*args)[2], (*args)[3]};
first_seen_ = false;
}
break;
+ case kRedisCmdLPush:
+ command_args = {"LPUSH", user_key, value.ToString()};
+ break;
+ case kRedisCmdRPush:
+ command_args = {"RPUSH", user_key, value.ToString()};
+ break;
case kRedisCmdLRem:
- // lrem will be parsed in deletecf, so ignore this putcf
+ // LREM will be parsed in DeleteCF, so ignore it here
+ break;
+ case kRedisCmdLMove:
+ // LMOVE will be parsed in DeleteCF, so ignore it here
break;
default:
- command_args = {cmd == kRedisCmdLPush ? "LPUSH" : "RPUSH", user_key, value.ToString()};
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Type=List: unhandled command with code "
+ << *parse_result;
}
break;
}
@@ -159,43 +187,53 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
}
case kRedisBitmap: {
auto args = log_data_.GetArguments();
- if (args->size() < 1) {
- LOG(ERROR) << "Fail to parse write_batch in putcf type bitmap : args error ,should at least contain cmd";
+ if (args->empty()) {
+ LOG(ERROR)
+ << "Failed to parse write_batch in PutCF. Type=Bitmap: no arguments, at least should contain a command";
return rocksdb::Status::OK();
}
- auto parse_result = ParseInt<int>((*args)[0], 10);
- if (!parse_result) {
- return rocksdb::Status::InvalidArgument(parse_result.Msg());
+
+ auto parsed_cmd = ParseInt<int>((*args)[0], 10);
+ if (!parsed_cmd) {
+ return rocksdb::Status::InvalidArgument(
+ fmt::format("failed to parse Redis command from log data: {}", parsed_cmd.Msg()));
}
- auto cmd = static_cast<RedisCommand>(*parse_result);
+
+ auto cmd = static_cast<RedisCommand>(*parsed_cmd);
switch (cmd) {
case kRedisCmdSetBit: {
if (args->size() < 2) {
- LOG(ERROR) << "Fail to parse write_batch in putcf cmd setbit : args error ,should contain setbit offset";
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Command=SETBIT: no enough arguments, should contain "
+ "an offset";
return rocksdb::Status::OK();
}
- auto parse_result = ParseInt<int>((*args)[1], 10);
- if (!parse_result) {
- return rocksdb::Status::InvalidArgument(parse_result.Msg());
+
+ auto parsed_offset = ParseInt<int>((*args)[1], 10);
+ if (!parsed_offset) {
+ return rocksdb::Status::InvalidArgument(
+ fmt::format("failed to parse an offset of SETBIT: {}", parsed_offset.Msg()));
}
- bool bit_value = Redis::Bitmap::GetBitFromValueAndOffset(value.ToString(), *parse_result);
+
+ bool bit_value = Redis::Bitmap::GetBitFromValueAndOffset(value.ToString(), *parsed_offset);
command_args = {"SETBIT", user_key, (*args)[1], bit_value ? "1" : "0"};
break;
}
case kRedisCmdBitOp:
if (first_seen_) {
if (args->size() < 4) {
- LOG(ERROR)
- << "Fail to parse write_batch in putcf cmd bitop : args error, should at least contain srckey";
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Command=BITOP: no enough arguments, at least "
+ "should contain srckey";
return rocksdb::Status::OK();
}
+
command_args = {"BITOP", (*args)[1], user_key};
command_args.insert(command_args.end(), args->begin() + 2, args->end());
first_seen_ = false;
}
break;
default:
- LOG(ERROR) << "Fail to parse write_batch in putcf type bitmap : cmd error";
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Type=Bitmap: unhandled command with code "
+ << *parsed_cmd;
return rocksdb::Status::OK();
}
break;
@@ -210,9 +248,9 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
break;
}
} else if (column_family_id == kColumnFamilyIDStream) {
- auto s = ExtractStreamAddCommand(is_slotid_encoded_, key, value, &command_args);
+ auto s = ExtractStreamAddCommand(is_slot_id_encoded_, key, value, &command_args);
if (!s.IsOK()) {
- LOG(ERROR) << "Fail to parse write_batch for the stream type: " << s.Msg();
+ LOG(ERROR) << "Failed to parse write_batch in PutCF. Type=Stream: " << s.Msg();
return rocksdb::Status::OK();
}
}
@@ -220,6 +258,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic
if (!command_args.empty()) {
resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args));
}
+
return rocksdb::Status::OK();
}
@@ -228,22 +267,28 @@ rocksdb::Status WriteBatchExtractor::DeleteCF(uint32_t column_family_id, const S
return rocksdb::Status::OK();
}
- std::string ns, user_key, sub_key;
std::vector<std::string> command_args;
+ std::string ns;
+
if (column_family_id == kColumnFamilyIDMetadata) {
- ExtractNamespaceKey(key, &ns, &user_key, is_slotid_encoded_);
- if (slot_ >= 0) {
- if (static_cast<uint16_t>(slot_) != GetSlotNumFromKey(user_key)) return rocksdb::Status::OK();
+ std::string user_key;
+ ExtractNamespaceKey(key, &ns, &user_key, is_slot_id_encoded_);
+
+ if (slot_id_ >= 0 && static_cast<uint16_t>(slot_id_) != GetSlotIdFromKey(user_key)) {
+ return rocksdb::Status::OK();
}
+
command_args = {"DEL", user_key};
} else if (column_family_id == kColumnFamilyIDDefault) {
- InternalKey ikey(key, is_slotid_encoded_);
- user_key = ikey.GetKey().ToString();
- if (slot_ >= 0) {
- if (static_cast<uint16_t>(slot_) != GetSlotNumFromKey(user_key)) return rocksdb::Status::OK();
+ InternalKey ikey(key, is_slot_id_encoded_);
+ std::string user_key = ikey.GetKey().ToString();
+ if (slot_id_ >= 0 && static_cast<uint16_t>(slot_id_) != GetSlotIdFromKey(user_key)) {
+ return rocksdb::Status::OK();
}
- sub_key = ikey.GetSubKey().ToString();
+
+ std::string sub_key = ikey.GetSubKey().ToString();
ns = ikey.GetNamespace().ToString();
+
switch (log_data_.GetRedisType()) {
case kRedisHash:
command_args = {"HDEL", user_key, sub_key};
@@ -256,22 +301,28 @@ rocksdb::Status WriteBatchExtractor::DeleteCF(uint32_t column_family_id, const S
break;
case kRedisList: {
auto args = log_data_.GetArguments();
- if (args->size() < 1) {
- LOG(ERROR) << "Fail to parse write_batch in DeleteCF type list : args error ,should contain cmd";
+ if (args->empty()) {
+ LOG(ERROR)
+ << "Failed to parse write_batch in DeleteCF. Type=List: no arguments, at least should contain a command";
return rocksdb::Status::OK();
}
+
auto parse_result = ParseInt<int>((*args)[0], 10);
if (!parse_result) {
- return rocksdb::Status::InvalidArgument(parse_result.Msg());
+ return rocksdb::Status::InvalidArgument(
+ fmt::format("failed to parse Redis command from log data: {}", parse_result.Msg()));
}
+
auto cmd = static_cast<RedisCommand>(*parse_result);
switch (cmd) {
case kRedisCmdLTrim:
if (first_seen_) {
if (args->size() < 3) {
- LOG(ERROR) << "Fail to parse write_batch in DeleteCF cmd ltrim : args error ,should contain start,stop";
+ LOG(ERROR) << "Failed to parse write_batch in DeleteCF; Command=LTRIM: no enough arguments, should "
+ "contain start and stop";
return rocksdb::Status::OK();
}
+
command_args = {"LTRIM", user_key, (*args)[1], (*args)[2]};
first_seen_ = false;
}
@@ -279,15 +330,35 @@ rocksdb::Status WriteBatchExtractor::DeleteCF(uint32_t column_family_id, const S
case kRedisCmdLRem:
if (first_seen_) {
if (args->size() < 3) {
- LOG(ERROR) << "Fail to parse write_batch in DeleteCF cmd lrem : args error ,should contain count,value";
+ LOG(ERROR) << "Failed to parse write_batch in DeleteCF. Command=LREM: no enough arguments, should "
+ "contain count and value";
return rocksdb::Status::OK();
}
+
command_args = {"LREM", user_key, (*args)[1], (*args)[2]};
first_seen_ = false;
}
break;
+ case kRedisCmdLPop:
+ command_args = {"LPOP", user_key};
+ break;
+ case kRedisCmdRPop:
+ command_args = {"RPOP", user_key};
+ break;
+ case kRedisCmdLMove:
+ if (first_seen_) {
+ if (args->size() < 5) {
+ LOG(ERROR) << "Failed to parse write_batch in DeleteCF; Command=LMOVE: no enough arguments, should "
+ "contain source, destination and where/from arguments";
+ return rocksdb::Status::OK();
+ }
+ command_args = {"LMOVE", (*args)[1], (*args)[2], (*args)[3], (*args)[4]};
+ first_seen_ = false;
+ }
+ break;
default:
- command_args = {cmd == kRedisCmdLPop ? "LPOP" : "RPOP", user_key};
+ LOG(ERROR) << "Failed to parse write_batch in DeleteCF. Type=List: unhandled command with code "
+ << *parse_result;
}
break;
}
@@ -301,7 +372,7 @@ rocksdb::Status WriteBatchExtractor::DeleteCF(uint32_t column_family_id, const S
break;
}
} else if (column_family_id == kColumnFamilyIDStream) {
- InternalKey ikey(key, is_slotid_encoded_);
+ InternalKey ikey(key, is_slot_id_encoded_);
Slice encoded_id = ikey.GetSubKey();
Redis::StreamEntryID entry_id;
GetFixed64(&encoded_id, &entry_id.ms);
@@ -312,25 +383,28 @@ rocksdb::Status WriteBatchExtractor::DeleteCF(uint32_t column_family_id, const S
if (!command_args.empty()) {
resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args));
}
+
return rocksdb::Status::OK();
}
rocksdb::Status WriteBatchExtractor::DeleteRangeCF(uint32_t column_family_id, const Slice &begin_key,
const Slice &end_key) {
- // Do nothing about DeleteRange operations
+ // Do nothing with DeleteRange operations
return rocksdb::Status::OK();
}
-Status WriteBatchExtractor::ExtractStreamAddCommand(bool is_slotid_encoded, const Slice &subkey, const Slice &value,
+Status WriteBatchExtractor::ExtractStreamAddCommand(bool is_slot_id_encoded, const Slice &subkey, const Slice &value,
std::vector<std::string> *command_args) {
- InternalKey ikey(subkey, is_slotid_encoded);
+ InternalKey ikey(subkey, is_slot_id_encoded);
std::string user_key = ikey.GetKey().ToString();
*command_args = {"XADD", user_key};
+
std::vector<std::string> values;
auto s = Redis::DecodeRawStreamEntryValue(value.ToString(), &values);
if (!s.IsOK()) {
return s.Prefixed("failed to decode stream values");
}
+
Slice encoded_id = ikey.GetSubKey();
Redis::StreamEntryID entry_id;
GetFixed64(&encoded_id, &entry_id.ms);
@@ -338,5 +412,6 @@ Status WriteBatchExtractor::ExtractStreamAddCommand(bool is_slotid_encoded, cons
command_args->emplace_back(entry_id.ToString());
command_args->insert(command_args->end(), values.begin(), values.end());
+
return Status::OK();
}
diff --git a/src/storage/batch_extractor.h b/src/storage/batch_extractor.h
index 31c56fc4..aa53af15 100644
--- a/src/storage/batch_extractor.h
+++ b/src/storage/batch_extractor.h
@@ -19,6 +19,7 @@
*/
#pragma once
+
#include <map>
#include <string>
#include <vector>
@@ -28,26 +29,26 @@
#include "status.h"
#include "storage.h"
-// An extractor to extract update from raw writebatch
+// An extractor to extract update from raw write batch
class WriteBatchExtractor : public rocksdb::WriteBatch::Handler {
public:
- explicit WriteBatchExtractor(bool is_slotid_encoded, int16_t slot = -1, bool to_redis = false)
- : is_slotid_encoded_(is_slotid_encoded), slot_(slot), to_redis_(to_redis) {}
+ explicit WriteBatchExtractor(bool is_slot_id_encoded, int16_t slot_id = -1, bool to_redis = false)
+ : is_slot_id_encoded_(is_slot_id_encoded), slot_id_(slot_id), to_redis_(to_redis) {}
+
void LogData(const rocksdb::Slice &blob) override;
rocksdb::Status PutCF(uint32_t column_family_id, const Slice &key, const Slice &value) override;
-
rocksdb::Status DeleteCF(uint32_t column_family_id, const Slice &key) override;
rocksdb::Status DeleteRangeCF(uint32_t column_family_id, const Slice &begin_key, const Slice &end_key) override;
std::map<std::string, std::vector<std::string>> *GetRESPCommands() { return &resp_commands_; }
- static Status ExtractStreamAddCommand(bool is_slotid_encoded, const Slice &subkey, const Slice &value,
+ static Status ExtractStreamAddCommand(bool is_slot_id_encoded, const Slice &subkey, const Slice &value,
std::vector<std::string> *command_args);
private:
std::map<std::string, std::vector<std::string>> resp_commands_;
Redis::WriteBatchLogData log_data_;
bool first_seen_ = true;
- bool is_slotid_encoded_ = false;
- int slot_;
+ bool is_slot_id_encoded_ = false;
+ int slot_id_;
bool to_redis_;
};
diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc
index deb3b3be..f383fc01 100644
--- a/src/storage/redis_db.cc
+++ b/src/storage/redis_db.cc
@@ -242,7 +242,7 @@ rocksdb::Status Database::Scan(const std::string &cursor, uint64_t limit, const
AppendNamespacePrefix(cursor, &ns_cursor);
if (storage_->IsSlotIdEncoded()) {
- slot_start = cursor.empty() ? 0 : GetSlotNumFromKey(cursor);
+ slot_start = cursor.empty() ? 0 : GetSlotIdFromKey(cursor);
ComposeNamespaceKey(namespace_, "", &ns_prefix, false);
if (!prefix.empty()) {
PutFixed16(&ns_prefix, slot_start);
diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc
index 14e13140..8a9c564a 100644
--- a/src/storage/redis_metadata.cc
+++ b/src/storage/redis_metadata.cc
@@ -131,7 +131,7 @@ void ComposeNamespaceKey(const Slice &ns, const Slice &key, std::string *ns_key,
ns_key->append(ns.data(), ns.size());
if (slot_id_encoded) {
- auto slot_id = GetSlotNumFromKey(key.ToString());
+ auto slot_id = GetSlotIdFromKey(key.ToString());
PutFixed16(ns_key, slot_id);
}
diff --git a/src/types/redis_list.cc b/src/types/redis_list.cc
index 6ad826c4..ebe0ba85 100644
--- a/src/types/redis_list.cc
+++ b/src/types/redis_list.cc
@@ -496,7 +496,8 @@ rocksdb::Status List::lmoveOnSingleList(const rocksdb::Slice &src, bool src_left
}
auto batch = storage_->GetWriteBatchBase();
- WriteBatchLogData log_data(kRedisList, {std::to_string(kRedisCmdLMove)});
+ WriteBatchLogData log_data(kRedisList, {std::to_string(kRedisCmdLMove), src.ToString(), src.ToString(),
+ src_left ? "left" : "right", dst_left ? "left" : "right"});
batch->PutLogData(log_data.Encode());
batch->Delete(curr_sub_key);
@@ -547,7 +548,8 @@ rocksdb::Status List::lmoveOnTwoLists(const rocksdb::Slice &src, const rocksdb::
elem->clear();
auto batch = storage_->GetWriteBatchBase();
- WriteBatchLogData log_data(kRedisList, {std::to_string(kRedisCmdLMove)});
+ WriteBatchLogData log_data(kRedisList, {std::to_string(kRedisCmdLMove), src.ToString(), dst.ToString(),
+ src_left ? "left" : "right", dst_left ? "left" : "right"});
batch->PutLogData(log_data.Encode());
uint64_t src_index = src_left ? src_metadata.head : src_metadata.tail - 1;
diff --git a/tests/gocase/integration/slotmigrate/slotmigrate_test.go b/tests/gocase/integration/slotmigrate/slotmigrate_test.go
index 7040ae4e..8ca89012 100644
--- a/tests/gocase/integration/slotmigrate/slotmigrate_test.go
+++ b/tests/gocase/integration/slotmigrate/slotmigrate_test.go
@@ -876,6 +876,75 @@ func TestSlotMigrateDataType(t *testing.T) {
// write the migrated slot to destination server
require.NoError(t, rdb1.Set(ctx, k, "slot17_value1", 0).Err())
})
+
+ t.Run("MIGRATE - LMOVE (src and dst are different) via parsing WAL logs", func(t *testing.T) {
+ slot1 := 18
+
+ srcListName := fmt.Sprintf("list_src_{%s}", util.SlotTable[slot1])
+ dstListName := fmt.Sprintf("list_dst_{%s}", util.SlotTable[slot1])
+
+ require.NoError(t, rdb0.Del(ctx, srcListName).Err())
+ require.NoError(t, rdb0.Del(ctx, dstListName).Err())
+
+ require.NoError(t, rdb0.ConfigSet(ctx, "migrate-speed", "512").Err())
+ defer func() {
+ require.NoError(t, rdb0.ConfigSet(ctx, "migrate-speed", "4096").Err())
+ }()
+
+ for i := 0; i < 1000; i++ {
+ require.NoError(t, rdb0.RPush(ctx, srcListName, fmt.Sprintf("element%d", i)).Err())
+ }
+
+ require.Equal(t, "OK", rdb0.Do(ctx, "clusterx", "migrate", slot1, id1).Val())
+ requireMigrateState(t, rdb0, slot1, SlotMigrationStateStarted)
+
+ for i := 0; i < 10; i++ {
+ require.NoError(t, rdb0.LMove(ctx, srcListName, dstListName, "RIGHT", "LEFT").Err())
+ }
+ waitForMigrateState(t, rdb0, slot1, SlotMigrationStateSuccess)
+
+ require.ErrorContains(t, rdb0.RPush(ctx, srcListName, "element1000").Err(), "MOVED")
+ require.Equal(t, int64(10), rdb1.LLen(ctx, dstListName).Val())
+ require.Equal(t, "element990", rdb1.LIndex(ctx, dstListName, 0).Val())
+ require.Equal(t, "element999", rdb1.LIndex(ctx, dstListName, -1).Val())
+ require.Equal(t, int64(990), rdb1.LLen(ctx, srcListName).Val())
+ require.Equal(t, "element0", rdb1.LIndex(ctx, srcListName, 0).Val())
+ require.Equal(t, "element989", rdb1.LIndex(ctx, srcListName, -1).Val())
+ })
+
+ t.Run("MIGRATE - LMOVE (src and dst are the same) via parsing WAL logs", func(t *testing.T) {
+ slot1 := 19
+
+ srcListName := fmt.Sprintf("list_src_{%s}", util.SlotTable[slot1])
+
+ require.NoError(t, rdb0.Del(ctx, srcListName).Err())
+
+ require.NoError(t, rdb0.ConfigSet(ctx, "migrate-speed", "512").Err())
+ defer func() {
+ require.NoError(t, rdb0.ConfigSet(ctx, "migrate-speed", "4096").Err())
+ }()
+
+ srcLen := 1_000
+
+ for i := 0; i < srcLen; i++ {
+ require.NoError(t, rdb0.RPush(ctx, srcListName, fmt.Sprintf("element%d", i)).Err())
+ }
+
+ require.Equal(t, "OK", rdb0.Do(ctx, "clusterx", "migrate", slot1, id1).Val())
+ requireMigrateState(t, rdb0, slot1, SlotMigrationStateStarted)
+
+ for i := 0; i < 10; i++ {
+ require.NoError(t, rdb0.LMove(ctx, srcListName, srcListName, "RIGHT", "LEFT").Err())
+ }
+ waitForMigrateState(t, rdb0, slot1, SlotMigrationStateSuccess)
+
+ require.ErrorContains(t, rdb0.RPush(ctx, srcListName, "element1000").Err(), "MOVED")
+ require.Equal(t, int64(srcLen), rdb1.LLen(ctx, srcListName).Val())
+ require.EqualValues(t, []string{"element990", "element991", "element992", "element993", "element994"},
+ rdb1.LRange(ctx, srcListName, 0, 4).Val())
+ require.EqualValues(t, []string{"element985", "element986", "element987", "element988", "element989"},
+ rdb1.LRange(ctx, srcListName, -5, -1).Val())
+ })
}
func waitForMigrateState(t testing.TB, client *redis.Client, slot int, state SlotMigrationState) {