You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@mesos.apache.org by Jie Yu <yu...@gmail.com> on 2017/02/20 10:28:22 UTC
Re: mesos git commit: Upgrade leveldb to 1.19.
Do we have some benchmark numbers to vet this change? We have some replicated log benchmarks.
Sent from my iPhone
> On Feb 19, 2017, at 11:37 PM, haosdent@apache.org wrote:
>
> Repository: mesos
> Updated Branches:
> refs/heads/master 86ea327ae -> 74878e255
>
>
> Upgrade leveldb to 1.19.
>
> Leveldb in modern version is required to support s390x and arm64.
> It's also required to replace default byte-wise comparator with
> varint comparator in `src/log/leveldb.cpp`.
>
> Review: https://reviews.apache.org/r/51053/
>
>
> Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/74878e25
> Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/74878e25
> Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/74878e25
>
> Branch: refs/heads/master
> Commit: 74878e255bb099029dde2a03e0b1d22fecf16000
> Parents: 86ea327
> Author: Tomasz Janiszewski <ja...@gmail.com>
> Authored: Mon Feb 20 12:24:44 2017 +0800
> Committer: Haosdent Huang <ha...@apache.org>
> Committed: Mon Feb 20 15:36:54 2017 +0800
>
> ----------------------------------------------------------------------
> 3rdparty/Makefile.am | 4 +-
> 3rdparty/cmake/Mesos3rdpartyConfigure.cmake | 2 +-
> 3rdparty/cmake/Versions.cmake | 2 +-
> 3rdparty/leveldb-1.19.patch | 30 ++++++++++++
> 3rdparty/leveldb-1.19.tar.gz | Bin 0 -> 220839 bytes
> 3rdparty/leveldb-1.4.patch | 57 -----------------------
> 3rdparty/leveldb-1.4.tar.gz | Bin 198113 -> 0 bytes
> 3rdparty/versions.am | 2 +-
> LICENSE | 2 +-
> src/Makefile.am | 2 +-
> src/python/native_common/ext_modules.py.in | 4 +-
> support/mesos-tidy/entrypoint.sh | 2 +-
> 12 files changed, 40 insertions(+), 67 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/Makefile.am
> ----------------------------------------------------------------------
> diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
> index bbf9cfe..61d832b2 100644
> --- a/3rdparty/Makefile.am
> +++ b/3rdparty/Makefile.am
> @@ -296,11 +296,11 @@ BUILT_SOURCES += $(nodist_libgmock_la_SOURCES)
> if WITH_BUNDLED_LEVELDB
> # TODO(charles): Figure out PIC options in our configure.ac or create
> # a configure.ac for leveldb.
> -$(LEVELDB)/libleveldb.a: $(LEVELDB)-stamp
> +$(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
> cd $(LEVELDB) && \
> $(MAKE) $(AM_MAKEFLAGS) CC="$(CC)" CXX="$(CXX)" OPT="$(CXXFLAGS) -fPIC"
>
> -ALL_LOCAL += $(LEVELDB)/libleveldb.a
> +ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
> endif
>
> if WITH_BUNDLED_ZOOKEEPER
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> ----------------------------------------------------------------------
> diff --git a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> index eeb2786..c606526 100755
> --- a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> +++ b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> @@ -47,7 +47,7 @@ endif (NOT WIN32)
> # Convenience variables for "lflags", the symbols we pass to CMake to generate
> # things like `-L/path/to/glog` or `-lglog`.
> if (NOT WIN32)
> - set(LEVELDB_LFLAG ${LEVELDB_ROOT}/libleveldb.a)
> + set(LEVELDB_LFLAG ${LEVELDB_ROOT}/out-static/libleveldb.a)
> set(ZOOKEEPER_LFLAG ${ZOOKEEPER_LIB}/lib/libzookeeper_mt.a)
> else (NOT WIN32)
> set(ZOOKEEPER_LFLAG zookeeper)
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/cmake/Versions.cmake
> ----------------------------------------------------------------------
> diff --git a/3rdparty/cmake/Versions.cmake b/3rdparty/cmake/Versions.cmake
> index ad23f38..9127263 100644
> --- a/3rdparty/cmake/Versions.cmake
> +++ b/3rdparty/cmake/Versions.cmake
> @@ -4,7 +4,7 @@ set(ELFIO_VERSION "3.2")
> set(GLOG_VERSION "0.3.3")
> set(GMOCK_VERSION "1.7.0")
> set(HTTP_PARSER_VERSION "2.6.2")
> -set(LEVELDB_VERSION "1.4")
> +set(LEVELDB_VERSION "1.19")
> set(LIBAPR_VERSION "1.5.2")
> set(LIBEV_VERSION "4.22")
> # TODO(hausdorff): (MESOS-3529) transition this back to a non-beta version.
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.19.patch
> ----------------------------------------------------------------------
> diff --git a/3rdparty/leveldb-1.19.patch b/3rdparty/leveldb-1.19.patch
> new file mode 100644
> index 0000000..e281e95
> --- /dev/null
> +++ b/3rdparty/leveldb-1.19.patch
> @@ -0,0 +1,30 @@
> +diff --git a/build_detect_platform b/build_detect_platform
> +index bb76c4f..c403e91 100755
> +--- a/build_detect_platform
> ++++ b/build_detect_platform
> +@@ -190,24 +190,7 @@ EOF
> + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> + fi
> +
> +- # Test whether Snappy library is installed
> +- # http://code.google.com/p/snappy/
> +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
> +- #include <snappy.h>
> +- int main() {}
> +-EOF
> +- if [ "$?" = 0 ]; then
> +- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> +- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
> +- fi
> +-
> +- # Test whether tcmalloc is available
> +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
> +- int main() {}
> +-EOF
> +- if [ "$?" = 0 ]; then
> +- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
> +- fi
> ++# Removed tests for TCMALLOC and SNAPPY to prevent additional dependencies.
> +
> + rm -f $CXXOUTPUT 2>/dev/null
> + fi
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.19.tar.gz
> ----------------------------------------------------------------------
> diff --git a/3rdparty/leveldb-1.19.tar.gz b/3rdparty/leveldb-1.19.tar.gz
> new file mode 100644
> index 0000000..6f15c06
> Binary files /dev/null and b/3rdparty/leveldb-1.19.tar.gz differ
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.4.patch
> ----------------------------------------------------------------------
> diff --git a/3rdparty/leveldb-1.4.patch b/3rdparty/leveldb-1.4.patch
> deleted file mode 100644
> index b899f01..0000000
> --- a/3rdparty/leveldb-1.4.patch
> +++ /dev/null
> @@ -1,57 +0,0 @@
> -diff --git a/build_detect_platform b/build_detect_platform
> -index b71bf02..864e5fb 100755
> ---- a/build_detect_platform
> -+++ b/build_detect_platform
> -@@ -128,24 +128,8 @@ EOF
> - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> - fi
> -
> -- # Test whether Snappy library is installed
> -- # http://code.google.com/p/snappy/
> -- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
> -- #include <snappy.h>
> -- int main() {}
> --EOF
> -- if [ "$?" = 0 ]; then
> -- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
> -- fi
> -+# Removed tests for TCMALLOC and SNAPPY to prevent additional dependencies.
> -
> -- # Test whether tcmalloc is available
> -- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
> -- int main() {}
> --EOF
> -- if [ "$?" = 0 ]; then
> -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
> -- fi
> - fi
> -
> - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
> -diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h
> -index 35ae550..b4769b3 100644
> ---- a/port/atomic_pointer.h
> -+++ b/port/atomic_pointer.h
> -@@ -36,6 +36,8 @@
> - #define ARCH_CPU_X86_FAMILY 1
> - #elif defined(__ARMEL__)
> - #define ARCH_CPU_ARM_FAMILY 1
> -+#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
> -+#define ARCH_CPU_PPC_FAMILY 1
> - #endif
> -
> - namespace leveldb {
> -@@ -83,6 +85,13 @@ inline void MemoryBarrier() {
> - }
> - #define LEVELDB_HAVE_MEMORY_BARRIER
> -
> -+// PPC
> -+#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
> -+inline void MemoryBarrier() {
> -+ asm volatile("lwsync" : : : "memory");
> -+}
> -+#define LEVELDB_HAVE_MEMORY_BARRIER
> -+
> - #endif
> -
> - // AtomicPointer built using platform-specific MemoryBarrier()
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.4.tar.gz
> ----------------------------------------------------------------------
> diff --git a/3rdparty/leveldb-1.4.tar.gz b/3rdparty/leveldb-1.4.tar.gz
> deleted file mode 100644
> index 2ddbc0c..0000000
> Binary files a/3rdparty/leveldb-1.4.tar.gz and /dev/null differ
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/versions.am
> ----------------------------------------------------------------------
> diff --git a/3rdparty/versions.am b/3rdparty/versions.am
> index 26f839c..b814470 100644
> --- a/3rdparty/versions.am
> +++ b/3rdparty/versions.am
> @@ -25,7 +25,7 @@ GLOG_VERSION = 0.3.3
> GMOCK_VERSION = 1.7.0
> GPERFTOOLS_VERSION = 2.5
> HTTP_PARSER_VERSION = 2.6.2
> -LEVELDB_VERSION = 1.4
> +LEVELDB_VERSION = 1.19
> LIBEV_VERSION = 4.22
> NVML_VERSION = 352.79
> PICOJSON_VERSION = 1.3.0
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/LICENSE
> ----------------------------------------------------------------------
> diff --git a/LICENSE b/LICENSE
> index 7946d1d..f11970c 100644
> --- a/LICENSE
> +++ b/LICENSE
> @@ -428,7 +428,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
>
>
> ======================================================================
> -For leveldb (3rdparty/leveldb-1.4.tar.gz):
> +For leveldb (3rdparty/leveldb-1.19.tar.gz):
> ======================================================================
>
> Copyright (c) 2011 The LevelDB Authors. All rights reserved.
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/src/Makefile.am
> ----------------------------------------------------------------------
> diff --git a/src/Makefile.am b/src/Makefile.am
> index c21a073..89fc72b 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -171,7 +171,7 @@ MESOS_CPPFLAGS += -I../$(LEVELDB)/include
> # the installed libmesos.la file to include leveldb in
> # 'dependency_libs' (via '-L../3rdparty/leveldb -lleveldb').
>
> -LIB_LEVELDB = ../$(LEVELDB)/libleveldb.a
> +LIB_LEVELDB = ../$(LEVELDB)/out-static/libleveldb.a
> else
> LIB_LEVELDB = -lleveldb
> LDADD += -lleveldb
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/src/python/native_common/ext_modules.py.in
> ----------------------------------------------------------------------
> diff --git a/src/python/native_common/ext_modules.py.in b/src/python/native_common/ext_modules.py.in
> index 2d4a45e..e0bb335 100644
> --- a/src/python/native_common/ext_modules.py.in
> +++ b/src/python/native_common/ext_modules.py.in
> @@ -31,7 +31,7 @@ def _create_module(module_name):
> ext_common_dir = os.path.join(
> 'src', 'python', 'native_common')
>
> - leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> + leveldb = os.path.join('3rdparty', 'leveldb-1.19')
> zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.8', 'src', 'c')
> libprocess = os.path.join('3rdparty', 'libprocess')
>
> @@ -74,7 +74,7 @@ def _create_module(module_name):
> # For leveldb, we need to check for the presence of libleveldb.a, since
> # it is possible to disable leveldb inside mesos.
> libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a')
> - libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a')
> + libleveldb = os.path.join(abs_top_builddir, leveldb, 'out-static', 'libleveldb.a')
> libzookeeper = os.path.join(
> abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> libprotobuf = os.path.join(
>
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/support/mesos-tidy/entrypoint.sh
> ----------------------------------------------------------------------
> diff --git a/support/mesos-tidy/entrypoint.sh b/support/mesos-tidy/entrypoint.sh
> index 4f03a60..5dbaa60 100755
> --- a/support/mesos-tidy/entrypoint.sh
> +++ b/support/mesos-tidy/entrypoint.sh
> @@ -45,7 +45,7 @@ cmake --build 3rdparty --target http_parser-2.6.2 -- -j $(nproc)
> cmake --build 3rdparty --target libev-4.22 -- -j $(nproc) || true
> cmake --build 3rdparty --target libevent-2.1.5-beta -- -j $(nproc) || true
>
> -cmake --build 3rdparty --target leveldb-1.4 -- -j $(nproc)
> +cmake --build 3rdparty --target leveldb-1.19 -- -j $(nproc)
> cmake --build 3rdparty --target nvml-352.79 -- -j $(nproc)
> cmake --build 3rdparty --target picojson-1.3.0 -- -j $(nproc)
> cmake --build 3rdparty --target protobuf-2.6.1 -- -j $(nproc)
>
Re: mesos git commit: Upgrade leveldb to 1.19.
Posted by Tomek Janiszewski <ja...@gmail.com>.
I run benchmarks and the results are here
https://gist.github.com/janisz/5d2585376fc8ffb6d0fa9bb25afc7b92
pon., 20 lut 2017 o 19:01 użytkownik haosdent <ha...@gmail.com> napisał:
> Thanks a lot for jie's great help. Create two tickets at
>
> https://issues.apache.org/jira/browse/MESOS-7147
> https://issues.apache.org/jira/browse/MESOS-7148
>
> Hopefully we would finish them in the following days to get the benchmark
> result.
> If the result shows leveldb 1.19 make the benchmark worst, I would revert
> the changes as the following step.
>
> On Tue, Feb 21, 2017 at 1:39 AM, Jie Yu <yu...@gmail.com> wrote:
>
> > Thanks Tomek!
> >
> > I don't think we have BENCHMARK test for replicated log, which uses
> leveldb
> > as its backend. We do have tools built to do manual testing.
> >
> > ```
> > Jies-MacBook-Pro:bin jie$ ./mesos-log --help
> > Cannot find command '--help'
> >
> > Usage: ./mesos-log <command> [OPTIONS]
> >
> > Available commands:
> > help
> > replica
> > read
> > initialize
> > benchmark
> > ```
> >
> > The `replica` command will launch a replica of the replicated log. The
> > `benchmark` will generate writes (either random or based on a trace) to
> the
> > replicated log, and report time. I think we should run this manually
> before
> > and after applying the patch to evaluate the write performance.
> >
> > We probably should add a BENCHMARK test to our test suite so that this
> can
> > be automated. For instance, initialize a replicated log with a single
> > replica, and perform a bunch of writes with random sizes.
> >
> > - Jie
> >
> >
> >
> > On Mon, Feb 20, 2017 at 3:13 AM, Tomek Janiszewski <ja...@gmail.com>
> > wrote:
> >
> > > Test summary and comparison between 1.4 and 1.19
> > > https://docs.google.com/document/d/1fv2OMvH6hVm6waacOejSrTJwUuDQe
> > > XlqqPDZjBmbcKU/edit#heading=h.ewfisffoslz3
> > >
> > > pon., 20 lut 2017 o 11:28 użytkownik Jie Yu <yu...@gmail.com>
> > napisał:
> > >
> > > > Do we have some benchmark numbers to vet this change? We have some
> > > > replicated log benchmarks.
> > > >
> > > > Sent from my iPhone
> > > >
> > > > > On Feb 19, 2017, at 11:37 PM, haosdent@apache.org wrote:
> > > > >
> > > > > Repository: mesos
> > > > > Updated Branches:
> > > > > refs/heads/master 86ea327ae -> 74878e255
> > > > >
> > > > >
> > > > > Upgrade leveldb to 1.19.
> > > > >
> > > > > Leveldb in modern version is required to support s390x and arm64.
> > > > > It's also required to replace default byte-wise comparator with
> > > > > varint comparator in `src/log/leveldb.cpp`.
> > > > >
> > > > > Review: https://reviews.apache.org/r/51053/
> > > > >
> > > > >
> > > > > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> > > > > Commit:
> http://git-wip-us.apache.org/repos/asf/mesos/commit/74878e25
> > > > > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/74878e25
> > > > > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/74878e25
> > > > >
> > > > > Branch: refs/heads/master
> > > > > Commit: 74878e255bb099029dde2a03e0b1d22fecf16000
> > > > > Parents: 86ea327
> > > > > Author: Tomasz Janiszewski <ja...@gmail.com>
> > > > > Authored: Mon Feb 20 12:24:44 2017 +0800
> > > > > Committer: Haosdent Huang <ha...@apache.org>
> > > > > Committed: Mon Feb 20 15:36:54 2017 +0800
> > > > >
> > > > > ------------------------------------------------------------
> > ----------
> > > > > 3rdparty/Makefile.am | 4 +-
> > > > > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake | 2 +-
> > > > > 3rdparty/cmake/Versions.cmake | 2 +-
> > > > > 3rdparty/leveldb-1.19.patch | 30 ++++++++++++
> > > > > 3rdparty/leveldb-1.19.tar.gz | Bin 0 -> 220839 bytes
> > > > > 3rdparty/leveldb-1.4.patch | 57
> > > -----------------------
> > > > > 3rdparty/leveldb-1.4.tar.gz | Bin 198113 -> 0 bytes
> > > > > 3rdparty/versions.am | 2 +-
> > > > > LICENSE | 2 +-
> > > > > src/Makefile.am | 2 +-
> > > > > src/python/native_common/ext_modules.py.in | 4 +-
> > > > > support/mesos-tidy/entrypoint.sh | 2 +-
> > > > > 12 files changed, 40 insertions(+), 67 deletions(-)
> > > > > ------------------------------------------------------------
> > ----------
> > > > >
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/Makefile.am
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
> > > > > index bbf9cfe..61d832b2 100644
> > > > > --- a/3rdparty/Makefile.am
> > > > > +++ b/3rdparty/Makefile.am
> > > > > @@ -296,11 +296,11 @@ BUILT_SOURCES +=
> $(nodist_libgmock_la_SOURCES)
> > > > > if WITH_BUNDLED_LEVELDB
> > > > > # TODO(charles): Figure out PIC options in our configure.ac or
> > create
> > > > > # a configure.ac for leveldb.
> > > > > -$(LEVELDB)/libleveldb.a: $(LEVELDB)-stamp
> > > > > +$(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
> > > > > cd $(LEVELDB) && \
> > > > > $(MAKE) $(AM_MAKEFLAGS) CC="$(CC)" CXX="$(CXX)"
> OPT="$(CXXFLAGS)
> > > > -fPIC"
> > > > >
> > > > > -ALL_LOCAL += $(LEVELDB)/libleveldb.a
> > > > > +ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
> > > > > endif
> > > > >
> > > > > if WITH_BUNDLED_ZOOKEEPER
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > > index eeb2786..c606526 100755
> > > > > --- a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > > +++ b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > > @@ -47,7 +47,7 @@ endif (NOT WIN32)
> > > > > # Convenience variables for "lflags", the symbols we pass to CMake
> to
> > > > generate
> > > > > # things like `-L/path/to/glog` or `-lglog`.
> > > > > if (NOT WIN32)
> > > > > - set(LEVELDB_LFLAG ${LEVELDB_ROOT}/libleveldb.a)
> > > > > + set(LEVELDB_LFLAG ${LEVELDB_ROOT}/out-static/libleveldb.a)
> > > > > set(ZOOKEEPER_LFLAG ${ZOOKEEPER_LIB}/lib/libzookeeper_mt.a)
> > > > > else (NOT WIN32)
> > > > > set(ZOOKEEPER_LFLAG zookeeper)
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/cmake/Versions.cmake
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/cmake/Versions.cmake
> > > > b/3rdparty/cmake/Versions.cmake
> > > > > index ad23f38..9127263 100644
> > > > > --- a/3rdparty/cmake/Versions.cmake
> > > > > +++ b/3rdparty/cmake/Versions.cmake
> > > > > @@ -4,7 +4,7 @@ set(ELFIO_VERSION "3.2")
> > > > > set(GLOG_VERSION "0.3.3")
> > > > > set(GMOCK_VERSION "1.7.0")
> > > > > set(HTTP_PARSER_VERSION "2.6.2")
> > > > > -set(LEVELDB_VERSION "1.4")
> > > > > +set(LEVELDB_VERSION "1.19")
> > > > > set(LIBAPR_VERSION "1.5.2")
> > > > > set(LIBEV_VERSION "4.22")
> > > > > # TODO(hausdorff): (MESOS-3529) transition this back to a non-beta
> > > > version.
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/leveldb-1.19.patch
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/leveldb-1.19.patch
> > b/3rdparty/leveldb-1.19.patch
> > > > > new file mode 100644
> > > > > index 0000000..e281e95
> > > > > --- /dev/null
> > > > > +++ b/3rdparty/leveldb-1.19.patch
> > > > > @@ -0,0 +1,30 @@
> > > > > +diff --git a/build_detect_platform b/build_detect_platform
> > > > > +index bb76c4f..c403e91 100755
> > > > > +--- a/build_detect_platform
> > > > > ++++ b/build_detect_platform
> > > > > +@@ -190,24 +190,7 @@ EOF
> > > > > + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > > > + fi
> > > > > +
> > > > > +- # Test whether Snappy library is installed
> > > > > +- # http://code.google.com/p/snappy/
> > > > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
> > > > > +- #include <snappy.h>
> > > > > +- int main() {}
> > > > > +-EOF
> > > > > +- if [ "$?" = 0 ]; then
> > > > > +- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > > > +- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
> > > > > +- fi
> > > > > +-
> > > > > +- # Test whether tcmalloc is available
> > > > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null
> > > <<EOF
> > > > > +- int main() {}
> > > > > +-EOF
> > > > > +- if [ "$?" = 0 ]; then
> > > > > +- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
> > > > > +- fi
> > > > > ++# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > > > dependencies.
> > > > > +
> > > > > + rm -f $CXXOUTPUT 2>/dev/null
> > > > > + fi
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/leveldb-1.19.tar.gz
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/leveldb-1.19.tar.gz
> > > b/3rdparty/leveldb-1.19.tar.gz
> > > > > new file mode 100644
> > > > > index 0000000..6f15c06
> > > > > Binary files /dev/null and b/3rdparty/leveldb-1.19.tar.gz differ
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/leveldb-1.4.patch
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/leveldb-1.4.patch
> b/3rdparty/leveldb-1.4.patch
> > > > > deleted file mode 100644
> > > > > index b899f01..0000000
> > > > > --- a/3rdparty/leveldb-1.4.patch
> > > > > +++ /dev/null
> > > > > @@ -1,57 +0,0 @@
> > > > > -diff --git a/build_detect_platform b/build_detect_platform
> > > > > -index b71bf02..864e5fb 100755
> > > > > ---- a/build_detect_platform
> > > > > -+++ b/build_detect_platform
> > > > > -@@ -128,24 +128,8 @@ EOF
> > > > > - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > > > - fi
> > > > > -
> > > > > -- # Test whether Snappy library is installed
> > > > > -- # http://code.google.com/p/snappy/
> > > > > -- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
> > > > > -- #include <snappy.h>
> > > > > -- int main() {}
> > > > > --EOF
> > > > > -- if [ "$?" = 0 ]; then
> > > > > -- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
> > > > > -- fi
> > > > > -+# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > > > dependencies.
> > > > > -
> > > > > -- # Test whether tcmalloc is available
> > > > > -- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null
> > <<EOF
> > > > > -- int main() {}
> > > > > --EOF
> > > > > -- if [ "$?" = 0 ]; then
> > > > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
> > > > > -- fi
> > > > > - fi
> > > > > -
> > > > > - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
> > > > > -diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h
> > > > > -index 35ae550..b4769b3 100644
> > > > > ---- a/port/atomic_pointer.h
> > > > > -+++ b/port/atomic_pointer.h
> > > > > -@@ -36,6 +36,8 @@
> > > > > - #define ARCH_CPU_X86_FAMILY 1
> > > > > - #elif defined(__ARMEL__)
> > > > > - #define ARCH_CPU_ARM_FAMILY 1
> > > > > -+#elif defined(__ppc__) || defined(__powerpc__) ||
> > > > defined(__powerpc64__)
> > > > > -+#define ARCH_CPU_PPC_FAMILY 1
> > > > > - #endif
> > > > > -
> > > > > - namespace leveldb {
> > > > > -@@ -83,6 +85,13 @@ inline void MemoryBarrier() {
> > > > > - }
> > > > > - #define LEVELDB_HAVE_MEMORY_BARRIER
> > > > > -
> > > > > -+// PPC
> > > > > -+#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
> > > > > -+inline void MemoryBarrier() {
> > > > > -+ asm volatile("lwsync" : : : "memory");
> > > > > -+}
> > > > > -+#define LEVELDB_HAVE_MEMORY_BARRIER
> > > > > -+
> > > > > - #endif
> > > > > -
> > > > > - // AtomicPointer built using platform-specific MemoryBarrier()
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/leveldb-1.4.tar.gz
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/leveldb-1.4.tar.gz
> > b/3rdparty/leveldb-1.4.tar.gz
> > > > > deleted file mode 100644
> > > > > index 2ddbc0c..0000000
> > > > > Binary files a/3rdparty/leveldb-1.4.tar.gz and /dev/null differ
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > 3rdparty/versions.am
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/3rdparty/versions.am b/3rdparty/versions.am
> > > > > index 26f839c..b814470 100644
> > > > > --- a/3rdparty/versions.am
> > > > > +++ b/3rdparty/versions.am
> > > > > @@ -25,7 +25,7 @@ GLOG_VERSION = 0.3.3
> > > > > GMOCK_VERSION = 1.7.0
> > > > > GPERFTOOLS_VERSION = 2.5
> > > > > HTTP_PARSER_VERSION = 2.6.2
> > > > > -LEVELDB_VERSION = 1.4
> > > > > +LEVELDB_VERSION = 1.19
> > > > > LIBEV_VERSION = 4.22
> > > > > NVML_VERSION = 352.79
> > > > > PICOJSON_VERSION = 1.3.0
> > > > >
> > > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/LICENSE
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/LICENSE b/LICENSE
> > > > > index 7946d1d..f11970c 100644
> > > > > --- a/LICENSE
> > > > > +++ b/LICENSE
> > > > > @@ -428,7 +428,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> > > POSSIBILITY
> > > > OF SUCH DAMAGE.
> > > > >
> > > > >
> > > > > ============================================================
> > ==========
> > > > > -For leveldb (3rdparty/leveldb-1.4.tar.gz):
> > > > > +For leveldb (3rdparty/leveldb-1.19.tar.gz):
> > > > > ============================================================
> > ==========
> > > > >
> > > > > Copyright (c) 2011 The LevelDB Authors. All rights reserved.
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > src/Makefile.am
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/src/Makefile.am b/src/Makefile.am
> > > > > index c21a073..89fc72b 100644
> > > > > --- a/src/Makefile.am
> > > > > +++ b/src/Makefile.am
> > > > > @@ -171,7 +171,7 @@ MESOS_CPPFLAGS += -I../$(LEVELDB)/include
> > > > > # the installed libmesos.la file to include leveldb in
> > > > > # 'dependency_libs' (via '-L../3rdparty/leveldb -lleveldb').
> > > > >
> > > > > -LIB_LEVELDB = ../$(LEVELDB)/libleveldb.a
> > > > > +LIB_LEVELDB = ../$(LEVELDB)/out-static/libleveldb.a
> > > > > else
> > > > > LIB_LEVELDB = -lleveldb
> > > > > LDADD += -lleveldb
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > src/python/native_common/ext_modules.py.in
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/src/python/native_common/ext_modules.py.in
> > > > b/src/python/native_common/ext_modules.py.in
> > > > > index 2d4a45e..e0bb335 100644
> > > > > --- a/src/python/native_common/ext_modules.py.in
> > > > > +++ b/src/python/native_common/ext_modules.py.in
> > > > > @@ -31,7 +31,7 @@ def _create_module(module_name):
> > > > > ext_common_dir = os.path.join(
> > > > > 'src', 'python', 'native_common')
> > > > >
> > > > > - leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> > > > > + leveldb = os.path.join('3rdparty', 'leveldb-1.19')
> > > > > zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.8', 'src',
> > 'c')
> > > > > libprocess = os.path.join('3rdparty', 'libprocess')
> > > > >
> > > > > @@ -74,7 +74,7 @@ def _create_module(module_name):
> > > > > # For leveldb, we need to check for the presence of
> libleveldb.a,
> > > > since
> > > > > # it is possible to disable leveldb inside mesos.
> > > > > libglog = os.path.join(abs_top_builddir, glog, '.libs',
> > > 'libglog.a')
> > > > > - libleveldb = os.path.join(abs_top_builddir, leveldb,
> > > 'libleveldb.a')
> > > > > + libleveldb = os.path.join(abs_top_builddir, leveldb,
> > 'out-static',
> > > > 'libleveldb.a')
> > > > > libzookeeper = os.path.join(
> > > > > abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> > > > > libprotobuf = os.path.join(
> > > > >
> > > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > > support/mesos-tidy/entrypoint.sh
> > > > > ------------------------------------------------------------
> > ----------
> > > > > diff --git a/support/mesos-tidy/entrypoint.sh
> > > > b/support/mesos-tidy/entrypoint.sh
> > > > > index 4f03a60..5dbaa60 100755
> > > > > --- a/support/mesos-tidy/entrypoint.sh
> > > > > +++ b/support/mesos-tidy/entrypoint.sh
> > > > > @@ -45,7 +45,7 @@ cmake --build 3rdparty --target http_parser-2.6.2
> > --
> > > > -j $(nproc)
> > > > > cmake --build 3rdparty --target libev-4.22 -- -j $(nproc) || true
> > > > > cmake --build 3rdparty --target libevent-2.1.5-beta -- -j $(nproc)
> ||
> > > > true
> > > > >
> > > > > -cmake --build 3rdparty --target leveldb-1.4 -- -j $(nproc)
> > > > > +cmake --build 3rdparty --target leveldb-1.19 -- -j $(nproc)
> > > > > cmake --build 3rdparty --target nvml-352.79 -- -j $(nproc)
> > > > > cmake --build 3rdparty --target picojson-1.3.0 -- -j $(nproc)
> > > > > cmake --build 3rdparty --target protobuf-2.6.1 -- -j $(nproc)
> > > > >
> > > >
> > >
> >
>
>
>
> --
> Best Regards,
> Haosdent Huang
>
Re: mesos git commit: Upgrade leveldb to 1.19.
Posted by haosdent <ha...@gmail.com>.
Thanks a lot for jie's great help. Create two tickets at
https://issues.apache.org/jira/browse/MESOS-7147
https://issues.apache.org/jira/browse/MESOS-7148
Hopefully we would finish them in the following days to get the benchmark
result.
If the result shows leveldb 1.19 make the benchmark worst, I would revert
the changes as the following step.
On Tue, Feb 21, 2017 at 1:39 AM, Jie Yu <yu...@gmail.com> wrote:
> Thanks Tomek!
>
> I don't think we have BENCHMARK test for replicated log, which uses leveldb
> as its backend. We do have tools built to do manual testing.
>
> ```
> Jies-MacBook-Pro:bin jie$ ./mesos-log --help
> Cannot find command '--help'
>
> Usage: ./mesos-log <command> [OPTIONS]
>
> Available commands:
> help
> replica
> read
> initialize
> benchmark
> ```
>
> The `replica` command will launch a replica of the replicated log. The
> `benchmark` will generate writes (either random or based on a trace) to the
> replicated log, and report time. I think we should run this manually before
> and after applying the patch to evaluate the write performance.
>
> We probably should add a BENCHMARK test to our test suite so that this can
> be automated. For instance, initialize a replicated log with a single
> replica, and perform a bunch of writes with random sizes.
>
> - Jie
>
>
>
> On Mon, Feb 20, 2017 at 3:13 AM, Tomek Janiszewski <ja...@gmail.com>
> wrote:
>
> > Test summary and comparison between 1.4 and 1.19
> > https://docs.google.com/document/d/1fv2OMvH6hVm6waacOejSrTJwUuDQe
> > XlqqPDZjBmbcKU/edit#heading=h.ewfisffoslz3
> >
> > pon., 20 lut 2017 o 11:28 użytkownik Jie Yu <yu...@gmail.com>
> napisał:
> >
> > > Do we have some benchmark numbers to vet this change? We have some
> > > replicated log benchmarks.
> > >
> > > Sent from my iPhone
> > >
> > > > On Feb 19, 2017, at 11:37 PM, haosdent@apache.org wrote:
> > > >
> > > > Repository: mesos
> > > > Updated Branches:
> > > > refs/heads/master 86ea327ae -> 74878e255
> > > >
> > > >
> > > > Upgrade leveldb to 1.19.
> > > >
> > > > Leveldb in modern version is required to support s390x and arm64.
> > > > It's also required to replace default byte-wise comparator with
> > > > varint comparator in `src/log/leveldb.cpp`.
> > > >
> > > > Review: https://reviews.apache.org/r/51053/
> > > >
> > > >
> > > > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> > > > Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/74878e25
> > > > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/74878e25
> > > > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/74878e25
> > > >
> > > > Branch: refs/heads/master
> > > > Commit: 74878e255bb099029dde2a03e0b1d22fecf16000
> > > > Parents: 86ea327
> > > > Author: Tomasz Janiszewski <ja...@gmail.com>
> > > > Authored: Mon Feb 20 12:24:44 2017 +0800
> > > > Committer: Haosdent Huang <ha...@apache.org>
> > > > Committed: Mon Feb 20 15:36:54 2017 +0800
> > > >
> > > > ------------------------------------------------------------
> ----------
> > > > 3rdparty/Makefile.am | 4 +-
> > > > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake | 2 +-
> > > > 3rdparty/cmake/Versions.cmake | 2 +-
> > > > 3rdparty/leveldb-1.19.patch | 30 ++++++++++++
> > > > 3rdparty/leveldb-1.19.tar.gz | Bin 0 -> 220839 bytes
> > > > 3rdparty/leveldb-1.4.patch | 57
> > -----------------------
> > > > 3rdparty/leveldb-1.4.tar.gz | Bin 198113 -> 0 bytes
> > > > 3rdparty/versions.am | 2 +-
> > > > LICENSE | 2 +-
> > > > src/Makefile.am | 2 +-
> > > > src/python/native_common/ext_modules.py.in | 4 +-
> > > > support/mesos-tidy/entrypoint.sh | 2 +-
> > > > 12 files changed, 40 insertions(+), 67 deletions(-)
> > > > ------------------------------------------------------------
> ----------
> > > >
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/Makefile.am
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
> > > > index bbf9cfe..61d832b2 100644
> > > > --- a/3rdparty/Makefile.am
> > > > +++ b/3rdparty/Makefile.am
> > > > @@ -296,11 +296,11 @@ BUILT_SOURCES += $(nodist_libgmock_la_SOURCES)
> > > > if WITH_BUNDLED_LEVELDB
> > > > # TODO(charles): Figure out PIC options in our configure.ac or
> create
> > > > # a configure.ac for leveldb.
> > > > -$(LEVELDB)/libleveldb.a: $(LEVELDB)-stamp
> > > > +$(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
> > > > cd $(LEVELDB) && \
> > > > $(MAKE) $(AM_MAKEFLAGS) CC="$(CC)" CXX="$(CXX)" OPT="$(CXXFLAGS)
> > > -fPIC"
> > > >
> > > > -ALL_LOCAL += $(LEVELDB)/libleveldb.a
> > > > +ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
> > > > endif
> > > >
> > > > if WITH_BUNDLED_ZOOKEEPER
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > index eeb2786..c606526 100755
> > > > --- a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > +++ b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > > @@ -47,7 +47,7 @@ endif (NOT WIN32)
> > > > # Convenience variables for "lflags", the symbols we pass to CMake to
> > > generate
> > > > # things like `-L/path/to/glog` or `-lglog`.
> > > > if (NOT WIN32)
> > > > - set(LEVELDB_LFLAG ${LEVELDB_ROOT}/libleveldb.a)
> > > > + set(LEVELDB_LFLAG ${LEVELDB_ROOT}/out-static/libleveldb.a)
> > > > set(ZOOKEEPER_LFLAG ${ZOOKEEPER_LIB}/lib/libzookeeper_mt.a)
> > > > else (NOT WIN32)
> > > > set(ZOOKEEPER_LFLAG zookeeper)
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/cmake/Versions.cmake
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/cmake/Versions.cmake
> > > b/3rdparty/cmake/Versions.cmake
> > > > index ad23f38..9127263 100644
> > > > --- a/3rdparty/cmake/Versions.cmake
> > > > +++ b/3rdparty/cmake/Versions.cmake
> > > > @@ -4,7 +4,7 @@ set(ELFIO_VERSION "3.2")
> > > > set(GLOG_VERSION "0.3.3")
> > > > set(GMOCK_VERSION "1.7.0")
> > > > set(HTTP_PARSER_VERSION "2.6.2")
> > > > -set(LEVELDB_VERSION "1.4")
> > > > +set(LEVELDB_VERSION "1.19")
> > > > set(LIBAPR_VERSION "1.5.2")
> > > > set(LIBEV_VERSION "4.22")
> > > > # TODO(hausdorff): (MESOS-3529) transition this back to a non-beta
> > > version.
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/leveldb-1.19.patch
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/leveldb-1.19.patch
> b/3rdparty/leveldb-1.19.patch
> > > > new file mode 100644
> > > > index 0000000..e281e95
> > > > --- /dev/null
> > > > +++ b/3rdparty/leveldb-1.19.patch
> > > > @@ -0,0 +1,30 @@
> > > > +diff --git a/build_detect_platform b/build_detect_platform
> > > > +index bb76c4f..c403e91 100755
> > > > +--- a/build_detect_platform
> > > > ++++ b/build_detect_platform
> > > > +@@ -190,24 +190,7 @@ EOF
> > > > + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > > + fi
> > > > +
> > > > +- # Test whether Snappy library is installed
> > > > +- # http://code.google.com/p/snappy/
> > > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
> > > > +- #include <snappy.h>
> > > > +- int main() {}
> > > > +-EOF
> > > > +- if [ "$?" = 0 ]; then
> > > > +- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > > +- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
> > > > +- fi
> > > > +-
> > > > +- # Test whether tcmalloc is available
> > > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null
> > <<EOF
> > > > +- int main() {}
> > > > +-EOF
> > > > +- if [ "$?" = 0 ]; then
> > > > +- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
> > > > +- fi
> > > > ++# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > > dependencies.
> > > > +
> > > > + rm -f $CXXOUTPUT 2>/dev/null
> > > > + fi
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/leveldb-1.19.tar.gz
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/leveldb-1.19.tar.gz
> > b/3rdparty/leveldb-1.19.tar.gz
> > > > new file mode 100644
> > > > index 0000000..6f15c06
> > > > Binary files /dev/null and b/3rdparty/leveldb-1.19.tar.gz differ
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/leveldb-1.4.patch
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/leveldb-1.4.patch b/3rdparty/leveldb-1.4.patch
> > > > deleted file mode 100644
> > > > index b899f01..0000000
> > > > --- a/3rdparty/leveldb-1.4.patch
> > > > +++ /dev/null
> > > > @@ -1,57 +0,0 @@
> > > > -diff --git a/build_detect_platform b/build_detect_platform
> > > > -index b71bf02..864e5fb 100755
> > > > ---- a/build_detect_platform
> > > > -+++ b/build_detect_platform
> > > > -@@ -128,24 +128,8 @@ EOF
> > > > - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > > - fi
> > > > -
> > > > -- # Test whether Snappy library is installed
> > > > -- # http://code.google.com/p/snappy/
> > > > -- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
> > > > -- #include <snappy.h>
> > > > -- int main() {}
> > > > --EOF
> > > > -- if [ "$?" = 0 ]; then
> > > > -- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
> > > > -- fi
> > > > -+# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > > dependencies.
> > > > -
> > > > -- # Test whether tcmalloc is available
> > > > -- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null
> <<EOF
> > > > -- int main() {}
> > > > --EOF
> > > > -- if [ "$?" = 0 ]; then
> > > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
> > > > -- fi
> > > > - fi
> > > > -
> > > > - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
> > > > -diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h
> > > > -index 35ae550..b4769b3 100644
> > > > ---- a/port/atomic_pointer.h
> > > > -+++ b/port/atomic_pointer.h
> > > > -@@ -36,6 +36,8 @@
> > > > - #define ARCH_CPU_X86_FAMILY 1
> > > > - #elif defined(__ARMEL__)
> > > > - #define ARCH_CPU_ARM_FAMILY 1
> > > > -+#elif defined(__ppc__) || defined(__powerpc__) ||
> > > defined(__powerpc64__)
> > > > -+#define ARCH_CPU_PPC_FAMILY 1
> > > > - #endif
> > > > -
> > > > - namespace leveldb {
> > > > -@@ -83,6 +85,13 @@ inline void MemoryBarrier() {
> > > > - }
> > > > - #define LEVELDB_HAVE_MEMORY_BARRIER
> > > > -
> > > > -+// PPC
> > > > -+#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
> > > > -+inline void MemoryBarrier() {
> > > > -+ asm volatile("lwsync" : : : "memory");
> > > > -+}
> > > > -+#define LEVELDB_HAVE_MEMORY_BARRIER
> > > > -+
> > > > - #endif
> > > > -
> > > > - // AtomicPointer built using platform-specific MemoryBarrier()
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/leveldb-1.4.tar.gz
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/leveldb-1.4.tar.gz
> b/3rdparty/leveldb-1.4.tar.gz
> > > > deleted file mode 100644
> > > > index 2ddbc0c..0000000
> > > > Binary files a/3rdparty/leveldb-1.4.tar.gz and /dev/null differ
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > 3rdparty/versions.am
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/3rdparty/versions.am b/3rdparty/versions.am
> > > > index 26f839c..b814470 100644
> > > > --- a/3rdparty/versions.am
> > > > +++ b/3rdparty/versions.am
> > > > @@ -25,7 +25,7 @@ GLOG_VERSION = 0.3.3
> > > > GMOCK_VERSION = 1.7.0
> > > > GPERFTOOLS_VERSION = 2.5
> > > > HTTP_PARSER_VERSION = 2.6.2
> > > > -LEVELDB_VERSION = 1.4
> > > > +LEVELDB_VERSION = 1.19
> > > > LIBEV_VERSION = 4.22
> > > > NVML_VERSION = 352.79
> > > > PICOJSON_VERSION = 1.3.0
> > > >
> > > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/LICENSE
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/LICENSE b/LICENSE
> > > > index 7946d1d..f11970c 100644
> > > > --- a/LICENSE
> > > > +++ b/LICENSE
> > > > @@ -428,7 +428,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> > POSSIBILITY
> > > OF SUCH DAMAGE.
> > > >
> > > >
> > > > ============================================================
> ==========
> > > > -For leveldb (3rdparty/leveldb-1.4.tar.gz):
> > > > +For leveldb (3rdparty/leveldb-1.19.tar.gz):
> > > > ============================================================
> ==========
> > > >
> > > > Copyright (c) 2011 The LevelDB Authors. All rights reserved.
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > src/Makefile.am
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/src/Makefile.am b/src/Makefile.am
> > > > index c21a073..89fc72b 100644
> > > > --- a/src/Makefile.am
> > > > +++ b/src/Makefile.am
> > > > @@ -171,7 +171,7 @@ MESOS_CPPFLAGS += -I../$(LEVELDB)/include
> > > > # the installed libmesos.la file to include leveldb in
> > > > # 'dependency_libs' (via '-L../3rdparty/leveldb -lleveldb').
> > > >
> > > > -LIB_LEVELDB = ../$(LEVELDB)/libleveldb.a
> > > > +LIB_LEVELDB = ../$(LEVELDB)/out-static/libleveldb.a
> > > > else
> > > > LIB_LEVELDB = -lleveldb
> > > > LDADD += -lleveldb
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > src/python/native_common/ext_modules.py.in
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/src/python/native_common/ext_modules.py.in
> > > b/src/python/native_common/ext_modules.py.in
> > > > index 2d4a45e..e0bb335 100644
> > > > --- a/src/python/native_common/ext_modules.py.in
> > > > +++ b/src/python/native_common/ext_modules.py.in
> > > > @@ -31,7 +31,7 @@ def _create_module(module_name):
> > > > ext_common_dir = os.path.join(
> > > > 'src', 'python', 'native_common')
> > > >
> > > > - leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> > > > + leveldb = os.path.join('3rdparty', 'leveldb-1.19')
> > > > zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.8', 'src',
> 'c')
> > > > libprocess = os.path.join('3rdparty', 'libprocess')
> > > >
> > > > @@ -74,7 +74,7 @@ def _create_module(module_name):
> > > > # For leveldb, we need to check for the presence of libleveldb.a,
> > > since
> > > > # it is possible to disable leveldb inside mesos.
> > > > libglog = os.path.join(abs_top_builddir, glog, '.libs',
> > 'libglog.a')
> > > > - libleveldb = os.path.join(abs_top_builddir, leveldb,
> > 'libleveldb.a')
> > > > + libleveldb = os.path.join(abs_top_builddir, leveldb,
> 'out-static',
> > > 'libleveldb.a')
> > > > libzookeeper = os.path.join(
> > > > abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> > > > libprotobuf = os.path.join(
> > > >
> > > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> > support/mesos-tidy/entrypoint.sh
> > > > ------------------------------------------------------------
> ----------
> > > > diff --git a/support/mesos-tidy/entrypoint.sh
> > > b/support/mesos-tidy/entrypoint.sh
> > > > index 4f03a60..5dbaa60 100755
> > > > --- a/support/mesos-tidy/entrypoint.sh
> > > > +++ b/support/mesos-tidy/entrypoint.sh
> > > > @@ -45,7 +45,7 @@ cmake --build 3rdparty --target http_parser-2.6.2
> --
> > > -j $(nproc)
> > > > cmake --build 3rdparty --target libev-4.22 -- -j $(nproc) || true
> > > > cmake --build 3rdparty --target libevent-2.1.5-beta -- -j $(nproc) ||
> > > true
> > > >
> > > > -cmake --build 3rdparty --target leveldb-1.4 -- -j $(nproc)
> > > > +cmake --build 3rdparty --target leveldb-1.19 -- -j $(nproc)
> > > > cmake --build 3rdparty --target nvml-352.79 -- -j $(nproc)
> > > > cmake --build 3rdparty --target picojson-1.3.0 -- -j $(nproc)
> > > > cmake --build 3rdparty --target protobuf-2.6.1 -- -j $(nproc)
> > > >
> > >
> >
>
--
Best Regards,
Haosdent Huang
Re: mesos git commit: Upgrade leveldb to 1.19.
Posted by Jie Yu <yu...@gmail.com>.
Thanks Tomek!
I don't think we have BENCHMARK test for replicated log, which uses leveldb
as its backend. We do have tools built to do manual testing.
```
Jies-MacBook-Pro:bin jie$ ./mesos-log --help
Cannot find command '--help'
Usage: ./mesos-log <command> [OPTIONS]
Available commands:
help
replica
read
initialize
benchmark
```
The `replica` command will launch a replica of the replicated log. The
`benchmark` will generate writes (either random or based on a trace) to the
replicated log, and report time. I think we should run this manually before
and after applying the patch to evaluate the write performance.
We probably should add a BENCHMARK test to our test suite so that this can
be automated. For instance, initialize a replicated log with a single
replica, and perform a bunch of writes with random sizes.
- Jie
On Mon, Feb 20, 2017 at 3:13 AM, Tomek Janiszewski <ja...@gmail.com>
wrote:
> Test summary and comparison between 1.4 and 1.19
> https://docs.google.com/document/d/1fv2OMvH6hVm6waacOejSrTJwUuDQe
> XlqqPDZjBmbcKU/edit#heading=h.ewfisffoslz3
>
> pon., 20 lut 2017 o 11:28 użytkownik Jie Yu <yu...@gmail.com> napisał:
>
> > Do we have some benchmark numbers to vet this change? We have some
> > replicated log benchmarks.
> >
> > Sent from my iPhone
> >
> > > On Feb 19, 2017, at 11:37 PM, haosdent@apache.org wrote:
> > >
> > > Repository: mesos
> > > Updated Branches:
> > > refs/heads/master 86ea327ae -> 74878e255
> > >
> > >
> > > Upgrade leveldb to 1.19.
> > >
> > > Leveldb in modern version is required to support s390x and arm64.
> > > It's also required to replace default byte-wise comparator with
> > > varint comparator in `src/log/leveldb.cpp`.
> > >
> > > Review: https://reviews.apache.org/r/51053/
> > >
> > >
> > > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> > > Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/74878e25
> > > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/74878e25
> > > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/74878e25
> > >
> > > Branch: refs/heads/master
> > > Commit: 74878e255bb099029dde2a03e0b1d22fecf16000
> > > Parents: 86ea327
> > > Author: Tomasz Janiszewski <ja...@gmail.com>
> > > Authored: Mon Feb 20 12:24:44 2017 +0800
> > > Committer: Haosdent Huang <ha...@apache.org>
> > > Committed: Mon Feb 20 15:36:54 2017 +0800
> > >
> > > ----------------------------------------------------------------------
> > > 3rdparty/Makefile.am | 4 +-
> > > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake | 2 +-
> > > 3rdparty/cmake/Versions.cmake | 2 +-
> > > 3rdparty/leveldb-1.19.patch | 30 ++++++++++++
> > > 3rdparty/leveldb-1.19.tar.gz | Bin 0 -> 220839 bytes
> > > 3rdparty/leveldb-1.4.patch | 57
> -----------------------
> > > 3rdparty/leveldb-1.4.tar.gz | Bin 198113 -> 0 bytes
> > > 3rdparty/versions.am | 2 +-
> > > LICENSE | 2 +-
> > > src/Makefile.am | 2 +-
> > > src/python/native_common/ext_modules.py.in | 4 +-
> > > support/mesos-tidy/entrypoint.sh | 2 +-
> > > 12 files changed, 40 insertions(+), 67 deletions(-)
> > > ----------------------------------------------------------------------
> > >
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/Makefile.am
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
> > > index bbf9cfe..61d832b2 100644
> > > --- a/3rdparty/Makefile.am
> > > +++ b/3rdparty/Makefile.am
> > > @@ -296,11 +296,11 @@ BUILT_SOURCES += $(nodist_libgmock_la_SOURCES)
> > > if WITH_BUNDLED_LEVELDB
> > > # TODO(charles): Figure out PIC options in our configure.ac or create
> > > # a configure.ac for leveldb.
> > > -$(LEVELDB)/libleveldb.a: $(LEVELDB)-stamp
> > > +$(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
> > > cd $(LEVELDB) && \
> > > $(MAKE) $(AM_MAKEFLAGS) CC="$(CC)" CXX="$(CXX)" OPT="$(CXXFLAGS)
> > -fPIC"
> > >
> > > -ALL_LOCAL += $(LEVELDB)/libleveldb.a
> > > +ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
> > > endif
> > >
> > > if WITH_BUNDLED_ZOOKEEPER
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > index eeb2786..c606526 100755
> > > --- a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > +++ b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > > @@ -47,7 +47,7 @@ endif (NOT WIN32)
> > > # Convenience variables for "lflags", the symbols we pass to CMake to
> > generate
> > > # things like `-L/path/to/glog` or `-lglog`.
> > > if (NOT WIN32)
> > > - set(LEVELDB_LFLAG ${LEVELDB_ROOT}/libleveldb.a)
> > > + set(LEVELDB_LFLAG ${LEVELDB_ROOT}/out-static/libleveldb.a)
> > > set(ZOOKEEPER_LFLAG ${ZOOKEEPER_LIB}/lib/libzookeeper_mt.a)
> > > else (NOT WIN32)
> > > set(ZOOKEEPER_LFLAG zookeeper)
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/cmake/Versions.cmake
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/cmake/Versions.cmake
> > b/3rdparty/cmake/Versions.cmake
> > > index ad23f38..9127263 100644
> > > --- a/3rdparty/cmake/Versions.cmake
> > > +++ b/3rdparty/cmake/Versions.cmake
> > > @@ -4,7 +4,7 @@ set(ELFIO_VERSION "3.2")
> > > set(GLOG_VERSION "0.3.3")
> > > set(GMOCK_VERSION "1.7.0")
> > > set(HTTP_PARSER_VERSION "2.6.2")
> > > -set(LEVELDB_VERSION "1.4")
> > > +set(LEVELDB_VERSION "1.19")
> > > set(LIBAPR_VERSION "1.5.2")
> > > set(LIBEV_VERSION "4.22")
> > > # TODO(hausdorff): (MESOS-3529) transition this back to a non-beta
> > version.
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/leveldb-1.19.patch
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/leveldb-1.19.patch b/3rdparty/leveldb-1.19.patch
> > > new file mode 100644
> > > index 0000000..e281e95
> > > --- /dev/null
> > > +++ b/3rdparty/leveldb-1.19.patch
> > > @@ -0,0 +1,30 @@
> > > +diff --git a/build_detect_platform b/build_detect_platform
> > > +index bb76c4f..c403e91 100755
> > > +--- a/build_detect_platform
> > > ++++ b/build_detect_platform
> > > +@@ -190,24 +190,7 @@ EOF
> > > + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > + fi
> > > +
> > > +- # Test whether Snappy library is installed
> > > +- # http://code.google.com/p/snappy/
> > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
> > > +- #include <snappy.h>
> > > +- int main() {}
> > > +-EOF
> > > +- if [ "$?" = 0 ]; then
> > > +- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > +- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
> > > +- fi
> > > +-
> > > +- # Test whether tcmalloc is available
> > > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null
> <<EOF
> > > +- int main() {}
> > > +-EOF
> > > +- if [ "$?" = 0 ]; then
> > > +- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
> > > +- fi
> > > ++# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > dependencies.
> > > +
> > > + rm -f $CXXOUTPUT 2>/dev/null
> > > + fi
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/leveldb-1.19.tar.gz
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/leveldb-1.19.tar.gz
> b/3rdparty/leveldb-1.19.tar.gz
> > > new file mode 100644
> > > index 0000000..6f15c06
> > > Binary files /dev/null and b/3rdparty/leveldb-1.19.tar.gz differ
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/leveldb-1.4.patch
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/leveldb-1.4.patch b/3rdparty/leveldb-1.4.patch
> > > deleted file mode 100644
> > > index b899f01..0000000
> > > --- a/3rdparty/leveldb-1.4.patch
> > > +++ /dev/null
> > > @@ -1,57 +0,0 @@
> > > -diff --git a/build_detect_platform b/build_detect_platform
> > > -index b71bf02..864e5fb 100755
> > > ---- a/build_detect_platform
> > > -+++ b/build_detect_platform
> > > -@@ -128,24 +128,8 @@ EOF
> > > - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > > - fi
> > > -
> > > -- # Test whether Snappy library is installed
> > > -- # http://code.google.com/p/snappy/
> > > -- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
> > > -- #include <snappy.h>
> > > -- int main() {}
> > > --EOF
> > > -- if [ "$?" = 0 ]; then
> > > -- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
> > > -- fi
> > > -+# Removed tests for TCMALLOC and SNAPPY to prevent additional
> > dependencies.
> > > -
> > > -- # Test whether tcmalloc is available
> > > -- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
> > > -- int main() {}
> > > --EOF
> > > -- if [ "$?" = 0 ]; then
> > > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
> > > -- fi
> > > - fi
> > > -
> > > - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
> > > -diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h
> > > -index 35ae550..b4769b3 100644
> > > ---- a/port/atomic_pointer.h
> > > -+++ b/port/atomic_pointer.h
> > > -@@ -36,6 +36,8 @@
> > > - #define ARCH_CPU_X86_FAMILY 1
> > > - #elif defined(__ARMEL__)
> > > - #define ARCH_CPU_ARM_FAMILY 1
> > > -+#elif defined(__ppc__) || defined(__powerpc__) ||
> > defined(__powerpc64__)
> > > -+#define ARCH_CPU_PPC_FAMILY 1
> > > - #endif
> > > -
> > > - namespace leveldb {
> > > -@@ -83,6 +85,13 @@ inline void MemoryBarrier() {
> > > - }
> > > - #define LEVELDB_HAVE_MEMORY_BARRIER
> > > -
> > > -+// PPC
> > > -+#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
> > > -+inline void MemoryBarrier() {
> > > -+ asm volatile("lwsync" : : : "memory");
> > > -+}
> > > -+#define LEVELDB_HAVE_MEMORY_BARRIER
> > > -+
> > > - #endif
> > > -
> > > - // AtomicPointer built using platform-specific MemoryBarrier()
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/leveldb-1.4.tar.gz
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/leveldb-1.4.tar.gz b/3rdparty/leveldb-1.4.tar.gz
> > > deleted file mode 100644
> > > index 2ddbc0c..0000000
> > > Binary files a/3rdparty/leveldb-1.4.tar.gz and /dev/null differ
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> 3rdparty/versions.am
> > > ----------------------------------------------------------------------
> > > diff --git a/3rdparty/versions.am b/3rdparty/versions.am
> > > index 26f839c..b814470 100644
> > > --- a/3rdparty/versions.am
> > > +++ b/3rdparty/versions.am
> > > @@ -25,7 +25,7 @@ GLOG_VERSION = 0.3.3
> > > GMOCK_VERSION = 1.7.0
> > > GPERFTOOLS_VERSION = 2.5
> > > HTTP_PARSER_VERSION = 2.6.2
> > > -LEVELDB_VERSION = 1.4
> > > +LEVELDB_VERSION = 1.19
> > > LIBEV_VERSION = 4.22
> > > NVML_VERSION = 352.79
> > > PICOJSON_VERSION = 1.3.0
> > >
> > > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/LICENSE
> > > ----------------------------------------------------------------------
> > > diff --git a/LICENSE b/LICENSE
> > > index 7946d1d..f11970c 100644
> > > --- a/LICENSE
> > > +++ b/LICENSE
> > > @@ -428,7 +428,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> POSSIBILITY
> > OF SUCH DAMAGE.
> > >
> > >
> > > ======================================================================
> > > -For leveldb (3rdparty/leveldb-1.4.tar.gz):
> > > +For leveldb (3rdparty/leveldb-1.19.tar.gz):
> > > ======================================================================
> > >
> > > Copyright (c) 2011 The LevelDB Authors. All rights reserved.
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> src/Makefile.am
> > > ----------------------------------------------------------------------
> > > diff --git a/src/Makefile.am b/src/Makefile.am
> > > index c21a073..89fc72b 100644
> > > --- a/src/Makefile.am
> > > +++ b/src/Makefile.am
> > > @@ -171,7 +171,7 @@ MESOS_CPPFLAGS += -I../$(LEVELDB)/include
> > > # the installed libmesos.la file to include leveldb in
> > > # 'dependency_libs' (via '-L../3rdparty/leveldb -lleveldb').
> > >
> > > -LIB_LEVELDB = ../$(LEVELDB)/libleveldb.a
> > > +LIB_LEVELDB = ../$(LEVELDB)/out-static/libleveldb.a
> > > else
> > > LIB_LEVELDB = -lleveldb
> > > LDADD += -lleveldb
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> src/python/native_common/ext_modules.py.in
> > > ----------------------------------------------------------------------
> > > diff --git a/src/python/native_common/ext_modules.py.in
> > b/src/python/native_common/ext_modules.py.in
> > > index 2d4a45e..e0bb335 100644
> > > --- a/src/python/native_common/ext_modules.py.in
> > > +++ b/src/python/native_common/ext_modules.py.in
> > > @@ -31,7 +31,7 @@ def _create_module(module_name):
> > > ext_common_dir = os.path.join(
> > > 'src', 'python', 'native_common')
> > >
> > > - leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> > > + leveldb = os.path.join('3rdparty', 'leveldb-1.19')
> > > zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.8', 'src', 'c')
> > > libprocess = os.path.join('3rdparty', 'libprocess')
> > >
> > > @@ -74,7 +74,7 @@ def _create_module(module_name):
> > > # For leveldb, we need to check for the presence of libleveldb.a,
> > since
> > > # it is possible to disable leveldb inside mesos.
> > > libglog = os.path.join(abs_top_builddir, glog, '.libs',
> 'libglog.a')
> > > - libleveldb = os.path.join(abs_top_builddir, leveldb,
> 'libleveldb.a')
> > > + libleveldb = os.path.join(abs_top_builddir, leveldb, 'out-static',
> > 'libleveldb.a')
> > > libzookeeper = os.path.join(
> > > abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> > > libprotobuf = os.path.join(
> > >
> > >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/
> support/mesos-tidy/entrypoint.sh
> > > ----------------------------------------------------------------------
> > > diff --git a/support/mesos-tidy/entrypoint.sh
> > b/support/mesos-tidy/entrypoint.sh
> > > index 4f03a60..5dbaa60 100755
> > > --- a/support/mesos-tidy/entrypoint.sh
> > > +++ b/support/mesos-tidy/entrypoint.sh
> > > @@ -45,7 +45,7 @@ cmake --build 3rdparty --target http_parser-2.6.2 --
> > -j $(nproc)
> > > cmake --build 3rdparty --target libev-4.22 -- -j $(nproc) || true
> > > cmake --build 3rdparty --target libevent-2.1.5-beta -- -j $(nproc) ||
> > true
> > >
> > > -cmake --build 3rdparty --target leveldb-1.4 -- -j $(nproc)
> > > +cmake --build 3rdparty --target leveldb-1.19 -- -j $(nproc)
> > > cmake --build 3rdparty --target nvml-352.79 -- -j $(nproc)
> > > cmake --build 3rdparty --target picojson-1.3.0 -- -j $(nproc)
> > > cmake --build 3rdparty --target protobuf-2.6.1 -- -j $(nproc)
> > >
> >
>
Re: mesos git commit: Upgrade leveldb to 1.19.
Posted by Tomek Janiszewski <ja...@gmail.com>.
Test summary and comparison between 1.4 and 1.19
https://docs.google.com/document/d/1fv2OMvH6hVm6waacOejSrTJwUuDQeXlqqPDZjBmbcKU/edit#heading=h.ewfisffoslz3
pon., 20 lut 2017 o 11:28 użytkownik Jie Yu <yu...@gmail.com> napisał:
> Do we have some benchmark numbers to vet this change? We have some
> replicated log benchmarks.
>
> Sent from my iPhone
>
> > On Feb 19, 2017, at 11:37 PM, haosdent@apache.org wrote:
> >
> > Repository: mesos
> > Updated Branches:
> > refs/heads/master 86ea327ae -> 74878e255
> >
> >
> > Upgrade leveldb to 1.19.
> >
> > Leveldb in modern version is required to support s390x and arm64.
> > It's also required to replace default byte-wise comparator with
> > varint comparator in `src/log/leveldb.cpp`.
> >
> > Review: https://reviews.apache.org/r/51053/
> >
> >
> > Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
> > Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/74878e25
> > Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/74878e25
> > Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/74878e25
> >
> > Branch: refs/heads/master
> > Commit: 74878e255bb099029dde2a03e0b1d22fecf16000
> > Parents: 86ea327
> > Author: Tomasz Janiszewski <ja...@gmail.com>
> > Authored: Mon Feb 20 12:24:44 2017 +0800
> > Committer: Haosdent Huang <ha...@apache.org>
> > Committed: Mon Feb 20 15:36:54 2017 +0800
> >
> > ----------------------------------------------------------------------
> > 3rdparty/Makefile.am | 4 +-
> > 3rdparty/cmake/Mesos3rdpartyConfigure.cmake | 2 +-
> > 3rdparty/cmake/Versions.cmake | 2 +-
> > 3rdparty/leveldb-1.19.patch | 30 ++++++++++++
> > 3rdparty/leveldb-1.19.tar.gz | Bin 0 -> 220839 bytes
> > 3rdparty/leveldb-1.4.patch | 57 -----------------------
> > 3rdparty/leveldb-1.4.tar.gz | Bin 198113 -> 0 bytes
> > 3rdparty/versions.am | 2 +-
> > LICENSE | 2 +-
> > src/Makefile.am | 2 +-
> > src/python/native_common/ext_modules.py.in | 4 +-
> > support/mesos-tidy/entrypoint.sh | 2 +-
> > 12 files changed, 40 insertions(+), 67 deletions(-)
> > ----------------------------------------------------------------------
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/Makefile.am
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
> > index bbf9cfe..61d832b2 100644
> > --- a/3rdparty/Makefile.am
> > +++ b/3rdparty/Makefile.am
> > @@ -296,11 +296,11 @@ BUILT_SOURCES += $(nodist_libgmock_la_SOURCES)
> > if WITH_BUNDLED_LEVELDB
> > # TODO(charles): Figure out PIC options in our configure.ac or create
> > # a configure.ac for leveldb.
> > -$(LEVELDB)/libleveldb.a: $(LEVELDB)-stamp
> > +$(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
> > cd $(LEVELDB) && \
> > $(MAKE) $(AM_MAKEFLAGS) CC="$(CC)" CXX="$(CXX)" OPT="$(CXXFLAGS)
> -fPIC"
> >
> > -ALL_LOCAL += $(LEVELDB)/libleveldb.a
> > +ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
> > endif
> >
> > if WITH_BUNDLED_ZOOKEEPER
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > index eeb2786..c606526 100755
> > --- a/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > +++ b/3rdparty/cmake/Mesos3rdpartyConfigure.cmake
> > @@ -47,7 +47,7 @@ endif (NOT WIN32)
> > # Convenience variables for "lflags", the symbols we pass to CMake to
> generate
> > # things like `-L/path/to/glog` or `-lglog`.
> > if (NOT WIN32)
> > - set(LEVELDB_LFLAG ${LEVELDB_ROOT}/libleveldb.a)
> > + set(LEVELDB_LFLAG ${LEVELDB_ROOT}/out-static/libleveldb.a)
> > set(ZOOKEEPER_LFLAG ${ZOOKEEPER_LIB}/lib/libzookeeper_mt.a)
> > else (NOT WIN32)
> > set(ZOOKEEPER_LFLAG zookeeper)
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/cmake/Versions.cmake
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/cmake/Versions.cmake
> b/3rdparty/cmake/Versions.cmake
> > index ad23f38..9127263 100644
> > --- a/3rdparty/cmake/Versions.cmake
> > +++ b/3rdparty/cmake/Versions.cmake
> > @@ -4,7 +4,7 @@ set(ELFIO_VERSION "3.2")
> > set(GLOG_VERSION "0.3.3")
> > set(GMOCK_VERSION "1.7.0")
> > set(HTTP_PARSER_VERSION "2.6.2")
> > -set(LEVELDB_VERSION "1.4")
> > +set(LEVELDB_VERSION "1.19")
> > set(LIBAPR_VERSION "1.5.2")
> > set(LIBEV_VERSION "4.22")
> > # TODO(hausdorff): (MESOS-3529) transition this back to a non-beta
> version.
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.19.patch
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/leveldb-1.19.patch b/3rdparty/leveldb-1.19.patch
> > new file mode 100644
> > index 0000000..e281e95
> > --- /dev/null
> > +++ b/3rdparty/leveldb-1.19.patch
> > @@ -0,0 +1,30 @@
> > +diff --git a/build_detect_platform b/build_detect_platform
> > +index bb76c4f..c403e91 100755
> > +--- a/build_detect_platform
> > ++++ b/build_detect_platform
> > +@@ -190,24 +190,7 @@ EOF
> > + COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > + fi
> > +
> > +- # Test whether Snappy library is installed
> > +- # http://code.google.com/p/snappy/
> > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
> > +- #include <snappy.h>
> > +- int main() {}
> > +-EOF
> > +- if [ "$?" = 0 ]; then
> > +- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > +- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
> > +- fi
> > +-
> > +- # Test whether tcmalloc is available
> > +- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
> > +- int main() {}
> > +-EOF
> > +- if [ "$?" = 0 ]; then
> > +- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
> > +- fi
> > ++# Removed tests for TCMALLOC and SNAPPY to prevent additional
> dependencies.
> > +
> > + rm -f $CXXOUTPUT 2>/dev/null
> > + fi
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.19.tar.gz
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/leveldb-1.19.tar.gz b/3rdparty/leveldb-1.19.tar.gz
> > new file mode 100644
> > index 0000000..6f15c06
> > Binary files /dev/null and b/3rdparty/leveldb-1.19.tar.gz differ
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.4.patch
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/leveldb-1.4.patch b/3rdparty/leveldb-1.4.patch
> > deleted file mode 100644
> > index b899f01..0000000
> > --- a/3rdparty/leveldb-1.4.patch
> > +++ /dev/null
> > @@ -1,57 +0,0 @@
> > -diff --git a/build_detect_platform b/build_detect_platform
> > -index b71bf02..864e5fb 100755
> > ---- a/build_detect_platform
> > -+++ b/build_detect_platform
> > -@@ -128,24 +128,8 @@ EOF
> > - COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
> > - fi
> > -
> > -- # Test whether Snappy library is installed
> > -- # http://code.google.com/p/snappy/
> > -- $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
> > -- #include <snappy.h>
> > -- int main() {}
> > --EOF
> > -- if [ "$?" = 0 ]; then
> > -- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
> > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
> > -- fi
> > -+# Removed tests for TCMALLOC and SNAPPY to prevent additional
> dependencies.
> > -
> > -- # Test whether tcmalloc is available
> > -- $CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null <<EOF
> > -- int main() {}
> > --EOF
> > -- if [ "$?" = 0 ]; then
> > -- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
> > -- fi
> > - fi
> > -
> > - PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
> > -diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h
> > -index 35ae550..b4769b3 100644
> > ---- a/port/atomic_pointer.h
> > -+++ b/port/atomic_pointer.h
> > -@@ -36,6 +36,8 @@
> > - #define ARCH_CPU_X86_FAMILY 1
> > - #elif defined(__ARMEL__)
> > - #define ARCH_CPU_ARM_FAMILY 1
> > -+#elif defined(__ppc__) || defined(__powerpc__) ||
> defined(__powerpc64__)
> > -+#define ARCH_CPU_PPC_FAMILY 1
> > - #endif
> > -
> > - namespace leveldb {
> > -@@ -83,6 +85,13 @@ inline void MemoryBarrier() {
> > - }
> > - #define LEVELDB_HAVE_MEMORY_BARRIER
> > -
> > -+// PPC
> > -+#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
> > -+inline void MemoryBarrier() {
> > -+ asm volatile("lwsync" : : : "memory");
> > -+}
> > -+#define LEVELDB_HAVE_MEMORY_BARRIER
> > -+
> > - #endif
> > -
> > - // AtomicPointer built using platform-specific MemoryBarrier()
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/leveldb-1.4.tar.gz
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/leveldb-1.4.tar.gz b/3rdparty/leveldb-1.4.tar.gz
> > deleted file mode 100644
> > index 2ddbc0c..0000000
> > Binary files a/3rdparty/leveldb-1.4.tar.gz and /dev/null differ
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/3rdparty/versions.am
> > ----------------------------------------------------------------------
> > diff --git a/3rdparty/versions.am b/3rdparty/versions.am
> > index 26f839c..b814470 100644
> > --- a/3rdparty/versions.am
> > +++ b/3rdparty/versions.am
> > @@ -25,7 +25,7 @@ GLOG_VERSION = 0.3.3
> > GMOCK_VERSION = 1.7.0
> > GPERFTOOLS_VERSION = 2.5
> > HTTP_PARSER_VERSION = 2.6.2
> > -LEVELDB_VERSION = 1.4
> > +LEVELDB_VERSION = 1.19
> > LIBEV_VERSION = 4.22
> > NVML_VERSION = 352.79
> > PICOJSON_VERSION = 1.3.0
> >
> > http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/LICENSE
> > ----------------------------------------------------------------------
> > diff --git a/LICENSE b/LICENSE
> > index 7946d1d..f11970c 100644
> > --- a/LICENSE
> > +++ b/LICENSE
> > @@ -428,7 +428,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
> OF SUCH DAMAGE.
> >
> >
> > ======================================================================
> > -For leveldb (3rdparty/leveldb-1.4.tar.gz):
> > +For leveldb (3rdparty/leveldb-1.19.tar.gz):
> > ======================================================================
> >
> > Copyright (c) 2011 The LevelDB Authors. All rights reserved.
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/src/Makefile.am
> > ----------------------------------------------------------------------
> > diff --git a/src/Makefile.am b/src/Makefile.am
> > index c21a073..89fc72b 100644
> > --- a/src/Makefile.am
> > +++ b/src/Makefile.am
> > @@ -171,7 +171,7 @@ MESOS_CPPFLAGS += -I../$(LEVELDB)/include
> > # the installed libmesos.la file to include leveldb in
> > # 'dependency_libs' (via '-L../3rdparty/leveldb -lleveldb').
> >
> > -LIB_LEVELDB = ../$(LEVELDB)/libleveldb.a
> > +LIB_LEVELDB = ../$(LEVELDB)/out-static/libleveldb.a
> > else
> > LIB_LEVELDB = -lleveldb
> > LDADD += -lleveldb
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/src/python/native_common/ext_modules.py.in
> > ----------------------------------------------------------------------
> > diff --git a/src/python/native_common/ext_modules.py.in
> b/src/python/native_common/ext_modules.py.in
> > index 2d4a45e..e0bb335 100644
> > --- a/src/python/native_common/ext_modules.py.in
> > +++ b/src/python/native_common/ext_modules.py.in
> > @@ -31,7 +31,7 @@ def _create_module(module_name):
> > ext_common_dir = os.path.join(
> > 'src', 'python', 'native_common')
> >
> > - leveldb = os.path.join('3rdparty', 'leveldb-1.4')
> > + leveldb = os.path.join('3rdparty', 'leveldb-1.19')
> > zookeeper = os.path.join('3rdparty', 'zookeeper-3.4.8', 'src', 'c')
> > libprocess = os.path.join('3rdparty', 'libprocess')
> >
> > @@ -74,7 +74,7 @@ def _create_module(module_name):
> > # For leveldb, we need to check for the presence of libleveldb.a,
> since
> > # it is possible to disable leveldb inside mesos.
> > libglog = os.path.join(abs_top_builddir, glog, '.libs', 'libglog.a')
> > - libleveldb = os.path.join(abs_top_builddir, leveldb, 'libleveldb.a')
> > + libleveldb = os.path.join(abs_top_builddir, leveldb, 'out-static',
> 'libleveldb.a')
> > libzookeeper = os.path.join(
> > abs_top_builddir, zookeeper, '.libs', 'libzookeeper_mt.a')
> > libprotobuf = os.path.join(
> >
> >
> http://git-wip-us.apache.org/repos/asf/mesos/blob/74878e25/support/mesos-tidy/entrypoint.sh
> > ----------------------------------------------------------------------
> > diff --git a/support/mesos-tidy/entrypoint.sh
> b/support/mesos-tidy/entrypoint.sh
> > index 4f03a60..5dbaa60 100755
> > --- a/support/mesos-tidy/entrypoint.sh
> > +++ b/support/mesos-tidy/entrypoint.sh
> > @@ -45,7 +45,7 @@ cmake --build 3rdparty --target http_parser-2.6.2 --
> -j $(nproc)
> > cmake --build 3rdparty --target libev-4.22 -- -j $(nproc) || true
> > cmake --build 3rdparty --target libevent-2.1.5-beta -- -j $(nproc) ||
> true
> >
> > -cmake --build 3rdparty --target leveldb-1.4 -- -j $(nproc)
> > +cmake --build 3rdparty --target leveldb-1.19 -- -j $(nproc)
> > cmake --build 3rdparty --target nvml-352.79 -- -j $(nproc)
> > cmake --build 3rdparty --target picojson-1.3.0 -- -j $(nproc)
> > cmake --build 3rdparty --target protobuf-2.6.1 -- -j $(nproc)
> >
>