You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by hu...@apache.org on 2022/06/23 11:19:25 UTC
[incubator-kvrocks] branch unstable updated: Support to use lz4 compression in RocksDB(#584)
This is an automated email from the ASF dual-hosted git repository.
hulk pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/incubator-kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new d42f029 Support to use lz4 compression in RocksDB(#584)
d42f029 is described below
commit d42f02969636c5006112147df7041d3b391b27b5
Author: xiaobiaozhao <52...@users.noreply.github.com>
AuthorDate: Thu Jun 23 19:19:20 2022 +0800
Support to use lz4 compression in RocksDB(#584)
RocksDB supported many compression libraries(like snappy/lz4/zstd...) to compress the user data,
but static linked all was not a good idea since it will import too many dependencies. With this concern,
Kvrocks supported snappy only before this PR and found the LZ4 performances better in many scenarios,
so decided to introduce the it now to make users happy.
Co-authored-by: Twice <tw...@gmail.com>
Co-authored-by: hulk <hu...@gmail.com>
---
CMakeLists.txt | 10 ++++++----
cmake/lz4.cmake | 37 +++++++++++++++++++++++++++++++++++++
cmake/rocksdb.cmake | 4 ++++
kvrocks.conf | 14 +++++++-------
src/config.cc | 1 +
5 files changed, 55 insertions(+), 11 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fd20464..65f7bbc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -50,6 +50,7 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "do not build shared libs by default")
include(cmake/gtest.cmake)
include(cmake/glog.cmake)
include(cmake/snappy.cmake)
+include(cmake/lz4.cmake)
include(cmake/rocksdb.cmake)
include(cmake/libevent.cmake)
include(cmake/lua.cmake)
@@ -61,22 +62,23 @@ list(APPEND EXTERNAL_LIBS PRIVATE snappy)
list(APPEND EXTERNAL_LIBS PRIVATE rocksdb_with_headers)
list(APPEND EXTERNAL_LIBS PRIVATE event_with_headers)
list(APPEND EXTERNAL_LIBS PRIVATE lua)
+list(APPEND EXTERNAL_LIBS PRIVATE lz4)
list(APPEND EXTERNAL_LIBS PRIVATE Threads::Threads)
# Add git sha to version.h
find_package(Git REQUIRED)
-execute_process(COMMAND sh -c "grep -i version -m1 Changelog | awk '{printf $3}'"
+execute_process(COMMAND sh -c "grep -i version -m1 Changelog | awk '{printf $3}'"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE PROJECT_VERSION)
-execute_process(COMMAND git rev-parse --short HEAD
+execute_process(COMMAND git rev-parse --short HEAD
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_SHA)
string(STRIP "${GIT_SHA}" GIT_SHA)
configure_file(src/version.h.in ${PROJECT_BINARY_DIR}/version.h)
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc")
-
+
if(ENABLE_STATIC_LIBSTDCXX)
- try_compile(FOUND_STATIC_LIBSTDCXX ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/checks/static_libstdcxx.cc
+ try_compile(FOUND_STATIC_LIBSTDCXX ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake/checks/static_libstdcxx.cc
LINK_OPTIONS -static-libstdc++ CXX_STANDARD 11)
if(NOT FOUND_STATIC_LIBSTDCXX)
diff --git a/cmake/lz4.cmake b/cmake/lz4.cmake
new file mode 100644
index 0000000..67d436f
--- /dev/null
+++ b/cmake/lz4.cmake
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard()
+include(FetchContent)
+
+FetchContent_Declare(
+ lz4
+ URL https://github.com/lz4/lz4/archive/v1.9.3.tar.gz
+ URL_HASH MD5=3a1ab1684e14fc1afc66228ce61b2db3
+)
+
+FetchContent_GetProperties(lz4)
+if(NOT lz4_POPULATED)
+ FetchContent_Populate(lz4)
+ add_custom_target(make_lz4 COMMAND make liblz4.a
+ WORKING_DIRECTORY ${lz4_SOURCE_DIR}/lib)
+endif()
+
+add_library(lz4 INTERFACE)
+target_include_directories(lz4 INTERFACE ${lz4_SOURCE_DIR}/lib)
+target_link_libraries(lz4 INTERFACE ${lz4_SOURCE_DIR}/lib/liblz4.a)
+add_dependencies(lz4 make_lz4)
diff --git a/cmake/rocksdb.cmake b/cmake/rocksdb.cmake
index 5817724..df85a8f 100644
--- a/cmake/rocksdb.cmake
+++ b/cmake/rocksdb.cmake
@@ -38,10 +38,14 @@ FetchContent_GetProperties(snappy)
FetchContent_MakeAvailableWithArgs(rocksdb
CMAKE_MODULE_PATH=${PROJECT_SOURCE_DIR}/cmake/modules # to locate FindJeMalloc.cmake
Snappy_DIR=${PROJECT_SOURCE_DIR}/cmake/modules # to locate SnappyConfig.cmake
+ lz4_ROOT_DIR=${lz4_SOURCE_DIR}/lib
+ lz4_LIBRARIES=${lz4_SOURCE_DIR}/lib/liblz4.a
+ lz4_INCLUDE_DIRS=${lz4_SOURCE_DIR}/lib
FAIL_ON_WARNINGS=OFF
WITH_TESTS=OFF
WITH_BENCHMARK_TOOLS=OFF
WITH_SNAPPY=ON
+ WITH_LZ4=ON
WITH_TOOLS=OFF
WITH_GFLAGS=OFF
USE_RTTI=ON
diff --git a/kvrocks.conf b/kvrocks.conf
index 9d4553c..cce97b9 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -523,7 +523,7 @@ rocksdb.cache_index_and_filter_blocks yes
# Specify the compression to use. Only compress level greater
# than 2 to improve performance.
-# Accept value: "no", "snappy"
+# Accept value: "no", "snappy", "lz4"
# default snappy
rocksdb.compression snappy
@@ -578,7 +578,7 @@ rocksdb.stats_dump_period_sec 0
# Default: no
rocksdb.disable_auto_compactions no
-# BlobDB(key-value separation) is essentially RocksDB for large-value use cases.
+# BlobDB(key-value separation) is essentially RocksDB for large-value use cases.
# Since 6.18.0, The new implementation is integrated into the RocksDB core.
# When set, large values (blobs) are written to separate blob files, and only
# pointers to them are stored in SST files. This can reduce write amplification
@@ -607,7 +607,7 @@ rocksdb.blob_file_size 268435456
# Enables garbage collection of blobs. Valid blobs residing in blob files
# older than a cutoff get relocated to new files as they are encountered
# during compaction, which makes it possible to clean up blob files once
-# they contain nothing but obsolete/garbage blobs.
+# they contain nothing but obsolete/garbage blobs.
# See also rocksdb.blob_garbage_collection_age_cutoff below.
#
# Default: yes
@@ -623,11 +623,11 @@ rocksdb.blob_garbage_collection_age_cutoff 25
# The purpose of following three options are to dynamically adjust the upper limit of
-# the data that each layer can store according to the size of the different
+# the data that each layer can store according to the size of the different
# layers of the LSM. Enabling this option will bring some improvements in
-# deletion efficiency and space amplification, but it will lose a certain
+# deletion efficiency and space amplification, but it will lose a certain
# amount of read performance.
-# If you want know more details about Levels' Target Size, you can read RocksDB wiki:
+# If you want know more details about Levels' Target Size, you can read RocksDB wiki:
# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size
#
# Default: no
@@ -636,7 +636,7 @@ rocksdb.level_compaction_dynamic_level_bytes no
# The total file size of level-1 sst.
#
# Default: 268435456 bytes
-rocksdb.max_bytes_for_level_base 268435456
+rocksdb.max_bytes_for_level_base 268435456
# Multiplication factor for the total file size of L(n+1) layers.
# This option is a double type number in RocksDB, but kvrocks is
diff --git a/src/config.cc b/src/config.cc
index cdd4f1f..6ea1d3e 100644
--- a/src/config.cc
+++ b/src/config.cc
@@ -47,6 +47,7 @@ const char *errNotSetLevelCompactionDynamicLevelBytes =
configEnum compression_type_enum[] = {
{"no", rocksdb::CompressionType::kNoCompression},
{"snappy", rocksdb::CompressionType::kSnappyCompression},
+ {"lz4", rocksdb::CompressionType::kLZ4Compression},
{nullptr, 0}
};