You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ab...@apache.org on 2019/06/01 08:38:12 UTC
[kudu] 02/04: [util] extract libnvm_cache from libkudu_util
This is an automated email from the ASF dual-hosted git repository.
abukor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit ff828ac91a335292c7c659fe3eb59a7f00cc7561
Author: Alexey Serbin <al...@apache.org>
AuthorDate: Wed May 29 22:28:16 2019 -0700
[util] extract libnvm_cache from libkudu_util
This patch extracts NVM/memkind-specific code from libkudu_util into
its own separate library: libnvm_cache. The motivation for this
change is to avoid linking NVM-backed LRU/FIFO cache code and libmemkind
into libraries and binaries which do not use that functionality,
but use DRAM-backed LRU/FIFO cache only.
A few follow-up changelists depend on this patch to introduce TTL cache
into DnsResolver.
Change-Id: Ibafea70676f7be9c086f9f195ab215cfff0719fe
Reviewed-on: http://gerrit.cloudera.org:8080/13468
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <ad...@cloudera.com>
---
src/kudu/cfile/CMakeLists.txt | 11 ++++++++++-
src/kudu/util/CMakeLists.txt | 39 ++++++++++++++++++++++++++-------------
src/kudu/util/cache.cc | 12 ------------
src/kudu/util/cache.h | 8 --------
src/kudu/util/nvm_cache.cc | 4 +++-
src/kudu/util/nvm_cache.h | 15 ++++++++-------
6 files changed, 47 insertions(+), 42 deletions(-)
diff --git a/src/kudu/cfile/CMakeLists.txt b/src/kudu/cfile/CMakeLists.txt
index 14887e1..e2cd369 100644
--- a/src/kudu/cfile/CMakeLists.txt
+++ b/src/kudu/cfile/CMakeLists.txt
@@ -45,7 +45,8 @@ add_library(cfile
index_btree.cc
type_encodings.cc)
-target_link_libraries(cfile
+
+set(CFILE_LIBS
kudu_common
kudu_fs
kudu_util
@@ -54,6 +55,14 @@ target_link_libraries(cfile
cfile_proto
bitshuffle)
+if(HAVE_LIB_MEMKIND)
+ set(CFILE_LIBS
+ ${CFILE_LIBS}
+ nvm_cache)
+endif()
+
+target_link_libraries(cfile ${CFILE_LIBS})
+
# Tests
SET_KUDU_TEST_LINK_LIBS(cfile)
ADD_KUDU_TEST(index-test)
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index a820114..a9a56ce 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -239,12 +239,6 @@ set(UTIL_SRCS
# optimized regardless of the default optimization options.
set_source_files_properties(memory/overwrite.cc PROPERTIES COMPILE_FLAGS "-O3")
-if(HAVE_LIB_MEMKIND)
- set(UTIL_SRCS
- ${UTIL_SRCS}
- nvm_cache.cc)
-endif()
-
set(UTIL_LIBS
crcutil
gflags
@@ -269,12 +263,6 @@ if(NOT APPLE)
rt)
endif()
-if(HAVE_LIB_MEMKIND)
- set(UTIL_LIBS
- ${UTIL_LIBS}
- memkind)
-endif()
-
# We use MallocExtension, but not in the exported version of the library.
set(EXPORTED_UTIL_LIBS ${UTIL_LIBS})
if(${KUDU_TCMALLOC_AVAILABLE})
@@ -288,6 +276,31 @@ ADD_EXPORTABLE_LIBRARY(kudu_util
EXPORTED_DEPS ${EXPORTED_UTIL_LIBS})
#######################################
+# nvm_cache
+#######################################
+
+if(HAVE_LIB_MEMKIND)
+ set(NVM_CACHE_SRCS
+ nvm_cache.cc)
+
+ set(NVM_CACHE_LIBS
+ gflags
+ glog
+ gutil
+ memkind)
+
+ if(NOT APPLE)
+ set(NVM_CACHE_LIBS
+ ${NVM_CACHE_LIBS}
+ dl
+ rt)
+ endif()
+
+ add_library(nvm_cache ${NVM_CACHE_SRCS})
+ target_link_libraries(nvm_cache ${NVM_CACHE_LIBS})
+endif()
+
+#######################################
# kudu_util_compression
#######################################
set(UTIL_COMPRESSION_SRCS
@@ -342,7 +355,7 @@ target_link_libraries(kudu_test_util
if(HAVE_LIB_MEMKIND)
target_link_libraries(kudu_test_util
- memkind)
+ nvm_cache)
endif()
#######################################
diff --git a/src/kudu/util/cache.cc b/src/kudu/util/cache.cc
index 5fced3c..073fa51 100644
--- a/src/kudu/util/cache.cc
+++ b/src/kudu/util/cache.cc
@@ -35,10 +35,6 @@
#include "kudu/util/slice.h"
#include "kudu/util/test_util_prod.h"
-#if !defined(__APPLE__)
-#include "kudu/util/nvm_cache.h"
-#endif
-
// Useful in tests that require accurate cache capacity accounting.
DEFINE_bool(cache_force_single_shard, false,
"Override all cache implementations to use just one shard");
@@ -697,14 +693,6 @@ Cache* NewCache<Cache::EvictionPolicy::LRU,
return new ShardedCache<Cache::EvictionPolicy::LRU>(capacity, id);
}
-#if defined(HAVE_LIB_MEMKIND)
-template<>
-Cache* NewCache<Cache::EvictionPolicy::LRU,
- Cache::MemoryType::NVM>(size_t capacity, const std::string& id) {
- return NewLRUNvmCache(capacity, id);
-}
-#endif
-
std::ostream& operator<<(std::ostream& os, Cache::MemoryType mem_type) {
switch (mem_type) {
case Cache::MemoryType::DRAM:
diff --git a/src/kudu/util/cache.h b/src/kudu/util/cache.h
index 11535c8..ff3a309 100644
--- a/src/kudu/util/cache.h
+++ b/src/kudu/util/cache.h
@@ -357,14 +357,6 @@ template<>
Cache* NewCache<Cache::EvictionPolicy::LRU,
Cache::MemoryType::DRAM>(size_t capacity, const std::string& id);
-#if defined(HAVE_LIB_MEMKIND)
-// Create a new LRU cache with a fixed size capacity. This implementation
-// of Cache uses the least-recently-used eviction policy and stored in NVM.
-template<>
-Cache* NewCache<Cache::EvictionPolicy::LRU,
- Cache::MemoryType::NVM>(size_t capacity, const std::string& id);
-#endif
-
// A helper method to output cache memory type into ostream.
std::ostream& operator<<(std::ostream& os, Cache::MemoryType mem_type);
diff --git a/src/kudu/util/nvm_cache.cc b/src/kudu/util/nvm_cache.cc
index 208aee4..55adffd 100644
--- a/src/kudu/util/nvm_cache.cc
+++ b/src/kudu/util/nvm_cache.cc
@@ -632,7 +632,9 @@ class ShardedLRUCache : public Cache {
} // end anonymous namespace
-Cache* NewLRUNvmCache(size_t capacity, const std::string& id) {
+template<>
+Cache* NewCache<Cache::EvictionPolicy::LRU,
+ Cache::MemoryType::NVM>(size_t capacity, const std::string& id) {
// memkind_create_pmem() will fail if the capacity is too small, but with
// an inscrutable error. So, we'll check ourselves.
CHECK_GE(capacity, MEMKIND_PMEM_MIN_SIZE)
diff --git a/src/kudu/util/nvm_cache.h b/src/kudu/util/nvm_cache.h
index 9a65316..eebbc91 100644
--- a/src/kudu/util/nvm_cache.h
+++ b/src/kudu/util/nvm_cache.h
@@ -14,18 +14,19 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#ifndef KUDU_UTIL_NVM_CACHE_H_
-#define KUDU_UTIL_NVM_CACHE_H_
+#pragma once
#include <cstddef>
#include <string>
+#include "kudu/util/cache.h"
+
namespace kudu {
-class Cache;
-// Create a cache in persistent memory with the given capacity.
-Cache* NewLRUNvmCache(size_t capacity, const std::string& id);
+// Create a new LRU cache with a fixed size capacity. This implementation
+// of Cache uses the least-recently-used eviction policy and stored in NVM.
+template<>
+Cache* NewCache<Cache::EvictionPolicy::LRU,
+ Cache::MemoryType::NVM>(size_t capacity, const std::string& id);
} // namespace kudu
-
-#endif