You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@mesos.apache.org by GitBox <gi...@apache.org> on 2018/08/27 18:03:00 UTC
[GitHub] greggomann closed pull request #290: Integrated mesos fetcher with
libarchive.
greggomann closed pull request #290: Integrated mesos fetcher with libarchive.
URL: https://github.com/apache/mesos/pull/290
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
index 3837a31562..0eb5247704 100644
--- a/3rdparty/CMakeLists.txt
+++ b/3rdparty/CMakeLists.txt
@@ -24,6 +24,7 @@ endif ()
# TODO(andschwa): Move each URL to the dependency itself.
set(BOOST_URL ${FETCH_URL}/boost-${BOOST_VERSION}.tar.gz)
+set(BZIP2_URL ${FETCH_URL}/bzip2-${BZIP2_VERSION}.tar.gz)
set(CONCURRENTQUEUE_URL ${FETCH_URL}/concurrentqueue-${CONCURRENTQUEUE_VERSION}.tar.gz)
set(CSI_URL ${FETCH_URL}/csi-${CSI_VERSION}.tar.gz)
set(ELFIO_URL ${FETCH_URL}/elfio-${ELFIO_VERSION}.tar.gz)
@@ -35,8 +36,10 @@ set(JEMALLOC_URL ${FETCH_URL}/jemalloc-${JEMALLOC_VERSION}.tar.gz)
set(PICOJSON_URL ${FETCH_URL}/picojson-${PICOJSON_VERSION}.tar.gz)
set(PROTOBUF_URL ${FETCH_URL}/protobuf-${PROTOBUF_VERSION}.tar.gz)
set(LEVELDB_URL ${FETCH_URL}/leveldb-${LEVELDB_VERSION}.tar.gz)
+set(LIBARCHIVE_URL ${FETCH_URL}/libarchive-${LIBARCHIVE_VERSION}.tar.gz)
set(LIBEV_URL ${FETCH_URL}/libev-${LIBEV_VERSION}.tar.gz)
set(NVML_URL ${FETCH_URL}/nvml-${NVML_VERSION}.tar.gz)
+set(XZ_URL ${FETCH_URL}/xz-${XZ_VERSION}.tar.gz)
set(ZOOKEEPER_URL ${FETCH_URL}/zookeeper-${ZOOKEEPER_VERSION}.tar.gz)
# NOTE: libevent doesn't come rebundled, so this URL is always the same. But,
@@ -752,6 +755,84 @@ else ()
endif ()
+# bzip2: A Massively Spiffy Yet Delicately Unobtrusive Compression Library.
+# https://bzip.org
+##########################################################################
+if (WIN32)
+ EXTERNAL(bzip2 ${BZIP2_VERSION} ${CMAKE_CURRENT_BINARY_DIR})
+ add_library(bzip2 STATIC IMPORTED)
+ add_dependencies(bzip2 ${BZIP2_TARGET})
+
+ set_target_properties(
+ bzip2 PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_ROOT};${BZIP2_ROOT}-build")
+
+ set_target_properties(
+ bzip2 PROPERTIES
+ IMPORTED_LOCATION_DEBUG ${BZIP2_ROOT}-build/Debug/bzip2${BZIP2_SHARED}${LIBRARY_SUFFIX}
+ IMPORTED_LOCATION_RELEASE ${BZIP2_ROOT}-build/Release/bzip2${BZIP2_SHARED}${LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_DEBUG ${BZIP2_ROOT}-build/Debug/bzip2${CMAKE_IMPORT_LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_RELEASE ${BZIP2_ROOT}-build/Release/bzip2${CMAKE_IMPORT_LIBRARY_SUFFIX})
+
+ PATCH_CMD(BZIP2_PATCH_CMD bzip2-${BZIP2_VERSION}.patch)
+
+ MAKE_INCLUDE_DIR(bzip2)
+ GET_BYPRODUCTS(bzip2)
+
+ set(BZIP2_CMAKE_ARGS
+ ${CMAKE_FORWARD_ARGS}
+ -DBZIP2_INSTALL_DIR=${BZIP2_ROOT}-lib)
+
+ ExternalProject_Add(
+ ${BZIP2_TARGET}
+ PREFIX ${BZIP2_CMAKE_ROOT}
+ BUILD_BYPRODUCTS ${BZIP2_BYPRODUCTS}
+ PATCH_COMMAND ${BZIP2_PATCH_CMD}
+ CMAKE_ARGS ${BZIP2_CMAKE_ARGS}
+ INSTALL_DIR ${BZIP2_ROOT}-lib
+ URL ${BZIP2_URL}
+ URL_HASH ${BZIP2_HASH})
+endif ()
+
+# xz: A Massively Spiffy Yet Delicately Unobtrusive Compression Library.
+# https://tukaani.org/xz/
+##########################################################################
+if (WIN32)
+ EXTERNAL(xz ${XZ_VERSION} ${CMAKE_CURRENT_BINARY_DIR})
+ add_library(xz STATIC IMPORTED)
+ add_dependencies(xz ${XZ_TARGET})
+
+ set_target_properties(
+ xz PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES "${XZ_ROOT};${XZ_ROOT}-build")
+
+ set_target_properties(
+ xz PROPERTIES
+ IMPORTED_LOCATION_DEBUG ${XZ_ROOT}-build/Debug/lzma${XZ_SHARED}${LIBRARY_SUFFIX}
+ IMPORTED_LOCATION_RELEASE ${XZ_ROOT}-build/Release/lzma${XZ_SHARED}${LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_DEBUG ${XZ_ROOT}-build/Debug/lzma${CMAKE_IMPORT_LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_RELEASE ${XZ_ROOT}-build/Release/lzma${CMAKE_IMPORT_LIBRARY_SUFFIX})
+
+ PATCH_CMD(XZ_PATCH_CMD xz-${XZ_VERSION}.patch)
+
+ MAKE_INCLUDE_DIR(xz)
+ GET_BYPRODUCTS(xz)
+
+ set(XZ_CMAKE_ARGS
+ ${CMAKE_FORWARD_ARGS}
+ -DXZ_INSTALL_DIR=${XZ_ROOT}-lib)
+
+ ExternalProject_Add(
+ ${XZ_TARGET}
+ PREFIX ${XZ_CMAKE_ROOT}
+ BUILD_BYPRODUCTS ${XZ_BYPRODUCTS}
+ PATCH_COMMAND ${XZ_PATCH_CMD}
+ CMAKE_ARGS ${XZ_CMAKE_ARGS}
+ INSTALL_DIR ${XZ_ROOT}-lib
+ URL ${XZ_URL}
+ URL_HASH ${XZ_HASH})
+endif ()
+
# zlib: A Massively Spiffy Yet Delicately Unobtrusive Compression Library.
# https://zlib.net
##########################################################################
@@ -824,6 +905,67 @@ set_target_properties(
zlib PROPERTIES
INTERFACE_COMPILE_DEFINITIONS HAVE_LIBZ)
+# libarchive: Multi-format archive and compression library.
+# https://github.com/libarchive/libarchive
+#####################################
+if (WIN32 OR REBUNDLED)
+EXTERNAL(libarchive ${LIBARCHIVE_VERSION} ${CMAKE_CURRENT_BINARY_DIR})
+
+add_library(libarchive STATIC IMPORTED GLOBAL)
+add_dependencies(libarchive ${LIBARCHIVE_TARGET})
+
+set(LIBARCHIVE_CMAKE_ARGS
+ ${CMAKE_FORWARD_ARGS}
+ -DENABLE_ACL=OFF
+ -DENABLE_TAR=ON
+ -DENABLE_TEST=OFF
+ -DENABLE_NETTLE=OFF
+ -DENABLE_OPENSSL=OFF
+ -DENABLE_LIBXML2=OFF
+ -DENABLE_EXPAT=OFF
+ -DENABLE_PCREPOSIX=OFF
+ -DENABLE_CPIO=OFF
+ -DCMAKE_PREFIX_PATH=${ZLIB_ROOT}-lib@@${BZIP2_ROOT}-lib@@${XZ_ROOT}-lib
+ -DCMAKE_INSTALL_PREFIX=${LIBARCHIVE_ROOT}-build)
+
+PATCH_CMD(LIBARCHIVE_PATCH_CMD libarchive-${LIBARCHIVE_VERSION}.patch)
+#
+# Disable tests Windows (can't handle UNICODE compile options)
+if (WIN32)
+ list(APPEND LIBARCHIVE_CMAKE_ARGS
+ -DENABLE_TEST=OFF)
+ SET_target_properties(
+ libarchive PROPERTIES
+ IMPORTED_LOCATION ${LIBARCHIVE_ROOT}-build/lib/archive_static${LIBRARY_SUFFIX}
+ INTERFACE_INCLUDE_DIRECTORIES ${LIBARCHIVE_ROOT}-build/include)
+ set(LIBARCHIVE_DEPENDS bzip2 xz zlib)
+else ()
+ list(APPEND LIBARCHIVE_CMAKE_ARGS
+ -DENABLE_TEST=ON)
+ SET_target_properties(
+ libarchive PROPERTIES
+ IMPORTED_LOCATION ${LIBARCHIVE_ROOT}-build/lib/libarchive${CMAKE_STATIC_LIBRARY_SUFFIX}
+ INTERFACE_INCLUDE_DIRECTORIES ${LIBARCHIVE_ROOT}-build/include)
+ set(LIBARCHIVE_DEPENDS "")
+endif ()
+
+MAKE_INCLUDE_DIR(libarchive)
+GET_BYPRODUCTS(libarchive)
+
+ExternalProject_Add(
+ ${LIBARCHIVE_TARGET}
+ DEPENDS ${LIBARCHIVE_DEPENDS}
+ PREFIX ${LIBARCHIVE_CMAKE_ROOT}
+ BUILD_BYPRODUCTS ${LIBARCHIVE_BYPRODUCTS}
+ PATCH_COMMAND ${LIBARCHIVE_PATCH_CMD}
+ LIST_SEPARATOR @@
+ CMAKE_ARGS ${LIBARCHIVE_CMAKE_ARGS}
+ URL ${LIBARCHIVE_URL}
+ URL_HASH ${LIBARCHIVE_HASH})
+else ()
+ find_package(LibArchive REQUIRED)
+ find_library(libarchive libarchive ${LIBARCHIVE_LIBRARIES})
+endif ()
# Google Test: Google's C++ test framework (GoogleTest and GoogleMock).
# https://github.com/google/googletest
@@ -1254,7 +1396,6 @@ if (NOT WIN32)
URL_HASH ${LEVELDB_HASH})
endif ()
-
# In-tree dependencies.
#######################
add_subdirectory(stout)
diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
index 062df18643..ba018cf8a2 100644
--- a/3rdparty/Makefile.am
+++ b/3rdparty/Makefile.am
@@ -61,6 +61,7 @@ GTEST = $(GOOGLETEST)/googletest
HTTP_PARSER = http-parser-$(HTTP_PARSER_VERSION)
JEMALLOC = jemalloc-$(JEMALLOC_VERSION)
LEVELDB = leveldb-$(LEVELDB_VERSION)
+LIBARCHIVE = libarchive-$(LIBARCHIVE_VERSION)
LIBEV = libev-$(LIBEV_VERSION)
NVML = nvml-$(NVML_VERSION)
PIP = pip-$(PIP_VERSION)
@@ -82,6 +83,7 @@ EXTRA_DIST = \
$(HTTP_PARSER).tar.gz \
$(JEMALLOC).tar.gz \
$(LEVELDB).tar.gz \
+ $(LIBARCHIVE).tar.gz \
$(LIBEV).tar.gz \
$(NVML).tar.gz \
$(PIP).tar.gz \
@@ -121,6 +123,7 @@ CLEAN_EXTRACTED = \
$(HTTP_PARSER) \
$(JEMALLOC) \
$(LEVELDB) \
+ $(LIBARCHIVE) \
$(LIBEV) \
$(NVML) \
$(PIP) \
@@ -269,6 +272,22 @@ $(JEMALLOC)-build-stamp: $(JEMALLOC)-stamp
ALL_LOCAL += $(LIB_JEMALLOC)
endif
+if WITH_BUNDLED_LIBARCHIVE
+LIB_LIBARCHIVE = $(LIBARCHIVE)/.libs/libarchive.la
+
+$(LIB_LIBARCHIVE): $(LIBARCHIVE)-build-stamp
+
+$(LIBARCHIVE)-build-stamp: $(LIBARCHIVE)-stamp
+ cd $(LIBARCHIVE) && \
+ CPPFLAGS="" \
+ LDFLAGS="" \
+ ./configure
+ cd $(LIBARCHIVE) && $(MAKE) $(AM_MAKEFLAGS)
+ touch $@
+
+ALL_LOCAL += $(LIB_LIBARCHIVE)
+endif
+
if WITH_BUNDLED_LIBEV
$(LIBEV)/libev.la: $(LIBEV)-build-stamp
$(LIBEV)-build-stamp: $(LIBEV)-stamp
diff --git a/3rdparty/bzip2-1.0.6.patch b/3rdparty/bzip2-1.0.6.patch
new file mode 100644
index 0000000000..01f68c06d9
--- /dev/null
+++ b/3rdparty/bzip2-1.0.6.patch
@@ -0,0 +1,20 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+new file mode 100644
+index 0000000..339ce93
+--- /dev/null
++++ b/CMakeLists.txt
+@@ -0,0 +1,14 @@
++project ( bzip2 C )
++cmake_minimum_required(VERSION 3.8)
++
++add_definitions ( -D_FILE_OFFSET_BITS=64 )
++
++set ( BZIP2_SRCS blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c )
++
++add_library ( bzip2 STATIC ${BZIP2_SRCS} libbz2.def )
++
++install(TARGETS bzip2
++ DESTINATION ${BZIP2_INSTALL_DIR})
++
++install(FILES bzlib.h
++ DESTINATION ${BZIP2_INSTALL_DIR})
diff --git a/3rdparty/bzip2-1.0.6.tar.gz b/3rdparty/bzip2-1.0.6.tar.gz
new file mode 100644
index 0000000000..e47e9034c4
Binary files /dev/null and b/3rdparty/bzip2-1.0.6.tar.gz differ
diff --git a/3rdparty/cmake/Versions.cmake b/3rdparty/cmake/Versions.cmake
index 2828acd2aa..f65073b0d9 100644
--- a/3rdparty/cmake/Versions.cmake
+++ b/3rdparty/cmake/Versions.cmake
@@ -31,10 +31,16 @@ set(PICOJSON_VERSION "1.3.0")
set(PICOJSON_HASH "SHA256=056805CA2691798F5545935A14BB477F2E1D827C9FB862E6E449DBEA22801C7D")
set(PROTOBUF_VERSION "3.5.0")
set(PROTOBUF_HASH "SHA256=F046682F05C39605E1687DC37E2E0EEBCEB8298CA1B046D64EE00AE124924EBC")
+set(LIBARCHIVE_VERSION "3.3.2")
+set(LIBARCHIVE_HASH "SHA256=ed2dbd6954792b2c054ccf8ec4b330a54b85904a80cef477a1c74643ddafa0ce")
+set(BZIP2_VERSION "1.0.6")
+set(BZIP2_HASH "SHA256=A2848F34FCD5D6CF47DEF00461FCB528A0484D8EDEF8208D6D2E2909DC61D9CD")
+set(XZ_VERSION "5.2.3")
+set(XZ_HASH "SHA256=71928B357D0A09A12A4B4C5FAFCA8C31C19B0E7D3B8EBB19622E96F26DBF28CB")
set(ZLIB_VERSION "1.2.8")
set(ZLIB_HASH "SHA256=36658CB768A54C1D4DEC43C3116C27ED893E88B02ECFCB44F2166F9C0B7F2A0D")
-set(ZOOKEEPER_VERSION "3.4.8")
-set(ZOOKEEPER_HASH "SHA256=F10A0B51F45C4F64C1FE69EF713ABF9EB9571BC7385A82DA892E83BB6C965E90")
+set(ZOOKEEPER_VERSION "3.4.8")
+set(ZOOKEEPER_HASH "SHA256=F10A0B51F45C4F64C1FE69EF713ABF9EB9571BC7385A82DA892E83BB6C965E90")
# Platform-dependent versions.
if (NOT WIN32)
diff --git a/3rdparty/libarchive-3.3.2.patch b/3rdparty/libarchive-3.3.2.patch
new file mode 100644
index 0000000000..b926285b2e
--- /dev/null
+++ b/3rdparty/libarchive-3.3.2.patch
@@ -0,0 +1,48 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 73bf07b..8f565c6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -3,6 +3,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12 FATAL_ERROR)
+ #
+ PROJECT(libarchive C)
+ #
++
+ SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/cmake")
+ if(NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${libarchive_BINARY_DIR}/bin)
+@@ -165,8 +166,6 @@ IF (MSVC)
+ # Enable level 4 C4706: The test value in a conditional expression was the
+ # result of an assignment.
+ SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /we4706")
+- # /WX option is the same as gcc's -Werror option.
+- SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /WX")
+ # /Oi option enables built-in functions.
+ SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /Oi")
+ #################################################################
+@@ -181,7 +180,6 @@ OPTION(ENABLE_NETTLE "Enable use of Nettle" ON)
+ OPTION(ENABLE_OPENSSL "Enable use of OpenSSL" ON)
+ OPTION(ENABLE_LZO "Enable the use of the system LZO library if found" OFF)
+ OPTION(ENABLE_LZMA "Enable the use of the system LZMA library if found" ON)
+-
+ OPTION(ENABLE_ZLIB "Enable the use of the system ZLIB library if found" ON)
+ OPTION(ENABLE_BZip2 "Enable the use of the system BZip2 library if found" ON)
+ OPTION(ENABLE_LIBXML2 "Enable the use of the system libxml2 library if found" ON)
+@@ -190,7 +188,6 @@ OPTION(ENABLE_PCREPOSIX "Enable the use of the system PCREPOSIX library if found
+ OPTION(ENABLE_LibGCC "Enable the use of the system LibGCC library if found" ON)
+ # CNG is used for encrypt/decrypt Zip archives on Windows.
+ OPTION(ENABLE_CNG "Enable the use of CNG(Crypto Next Generation)" ON)
+-
+ OPTION(ENABLE_TAR "Enable tar building" ON)
+ OPTION(ENABLE_TAR_SHARED "Enable dynamic build of tar" FALSE)
+ OPTION(ENABLE_CPIO "Enable cpio building" ON)
+@@ -902,10 +899,6 @@ MACRO(CHECK_ICONV LIB TRY_ICONV_CONST)
+ IF (CMAKE_C_COMPILER_ID MATCHES "^XL$")
+ SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -qhalt=w -qflag=w:w")
+ ENDIF (CMAKE_C_COMPILER_ID MATCHES "^XL$")
+- IF (MSVC)
+- # NOTE: /WX option is the same as gcc's -Werror option.
+- SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /WX")
+- ENDIF (MSVC)
+ #
+ CHECK_C_SOURCE_COMPILES(
+ "#include <stdlib.h>
diff --git a/3rdparty/libarchive-3.3.2.tar.gz b/3rdparty/libarchive-3.3.2.tar.gz
new file mode 100644
index 0000000000..beb1d2132d
Binary files /dev/null and b/3rdparty/libarchive-3.3.2.tar.gz differ
diff --git a/3rdparty/stout/CMakeLists.txt b/3rdparty/stout/CMakeLists.txt
index 24a1f0acbe..c53e885d05 100644
--- a/3rdparty/stout/CMakeLists.txt
+++ b/3rdparty/stout/CMakeLists.txt
@@ -29,10 +29,11 @@ target_link_libraries(
picojson
protobuf
Threads::Threads
+ libarchive
zlib
$<$<PLATFORM_ID:Linux>:rt dl svn>
$<$<PLATFORM_ID:Darwin>:svn>
- $<$<PLATFORM_ID:Windows>:Ws2_32 IPHlpAPI Mswsock Secur32 Userenv>
+ $<$<PLATFORM_ID:Windows>:Ws2_32 IPHlpAPI Mswsock Secur32 Userenv bzip2 xz>
$<$<PLATFORM_ID:FreeBSD>:svn>)
add_subdirectory(tests)
diff --git a/3rdparty/stout/Makefile.am b/3rdparty/stout/Makefile.am
index ef22a02a8a..5b922af6cd 100644
--- a/3rdparty/stout/Makefile.am
+++ b/3rdparty/stout/Makefile.am
@@ -53,6 +53,7 @@ GLOG = $(BUNDLED_DIR)/glog-$(GLOG_VERSION)
GOOGLETEST = $(BUNDLED_DIR)/googletest-release-$(GOOGLETEST_VERSION)
GMOCK = $(GOOGLETEST)/googlemock
GTEST = $(GOOGLETEST)/googletest
+LIBARCHIVE = $(BUNDLED_DIR)/libarchive-$(LIBARCHIVE_VERSION)
PROTOBUF = $(BUNDLED_DIR)/protobuf-$(PROTOBUF_VERSION)
PICOJSON = $(BUNDLED_DIR)/picojson-$(PICOJSON_VERSION)
@@ -97,6 +98,13 @@ PICOJSON_INCLUDE_FLAGS = \
-DPICOJSON_USE_INT64 \
-D__STDC_FORMAT_MACROS
+if WITH_BUNDLED_LIBARCHIVE
+LIBARCHIVE_INCLUDE_FLAGS = -I$(LIBARCHIVE)/libarchive
+LIB_LIBARCHIVE = $(LIBARCHIVE)/.libs/libarchive.la
+else
+LIB_LIBARCHIVE = -larchive
+endif
+
if WITH_BUNDLED_PICOJSON
PICOJSON_INCLUDE_FLAGS += -I$(PICOJSON)
BUNDLED_DEPS += $(PICOJSON)-stamp
@@ -124,6 +132,7 @@ check_PROGRAMS = stout-tests
stout_tests_SOURCES = \
tests/adaptor_tests.cpp \
+ tests/archiver_tests.cpp \
tests/base64_tests.cpp \
tests/bits_tests.cpp \
tests/boundedhashmap_tests.cpp \
@@ -189,6 +198,7 @@ stout_tests_CPPFLAGS = \
$(GLOG_INCLUDE_FLAGS) \
$(GMOCK_INCLUDE_FLAGS) \
$(GTEST_INCLUDE_FLAGS) \
+ $(LIBARCHIVE_INCLUDE_FLAGS) \
$(PICOJSON_INCLUDE_FLAGS) \
$(PROTOBUF_INCLUDE_FLAGS) \
$(AM_CPPFLAGS)
@@ -199,6 +209,7 @@ stout_tests_CPPFLAGS = \
stout_tests_LDADD = \
$(LIB_GMOCK) \
$(LIB_GLOG) \
+ $(LIB_LIBARCHIVE) \
$(LIB_PROTOBUF) \
-lsvn_subr-1 \
-lsvn_delta-1 \
diff --git a/3rdparty/stout/include/stout/archiver.hpp b/3rdparty/stout/include/stout/archiver.hpp
new file mode 100644
index 0000000000..d025d9a262
--- /dev/null
+++ b/3rdparty/stout/include/stout/archiver.hpp
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_ARCHIVER_HPP__
+#define __STOUT_ARCHIVER_HPP__
+
+#define LIBARCHIVE_STATIC
+
+#include <archive.h>
+#include <archive_entry.h>
+
+#include <glog/logging.h>
+
+#include <stout/nothing.hpp>
+#include <stout/os/int_fd.hpp>
+#include <stout/os/open.hpp>
+#include <stout/os/close.hpp>
+#include <stout/path.hpp>
+#include <stout/try.hpp>
+
+namespace archiver {
+
+// Flags can be any of (or together):
+// ARCHIVE_EXTRACT_ACL
+// ARCHIVE_EXTRACT_FFLAGS
+// ARCHIVE_EXTRACT_PERM
+// ARCHIVE_EXTRACT_TIME
+
+static bool handleError(const int result,
+ struct archive *a, Try<Nothing> &returnStatus)
+{
+ if (result < ARCHIVE_OK) {
+ LOG(WARNING) << archive_error_string(a);
+ if (returnStatus.isSome()) {
+ returnStatus = Error("Archive extraction failed due to prior errors");
+ }
+ }
+ if (result < ARCHIVE_WARN)
+ {
+ returnStatus = Error(archive_error_string(a));
+ return true;
+ }
+ return false;
+}
+
+inline Try<Nothing> extract(
+ const std::string& source,
+ const std::string& destination,
+ const int flags = ARCHIVE_EXTRACT_TIME)
+{
+ Try<Nothing> returnStatus = Nothing();
+
+ // Get references to libarchive for reading/handling a compressed file.
+
+ std::unique_ptr<struct archive, std::function<void(struct archive*)>> reader(
+ archive_read_new(),
+ [](struct archive* p) { archive_read_close(p); archive_read_free(p); });
+
+ archive_read_support_format_all(reader.get());
+ archive_read_support_filter_all(reader.get());
+
+ std::unique_ptr<struct archive, std::function<void(struct archive*)>> writer(
+ archive_write_disk_new(),
+ [](struct archive* p) { archive_write_close(p); archive_write_free(p); });
+
+ archive_write_disk_set_options(writer.get(), flags);
+ archive_write_disk_set_standard_lookup(writer.get());
+
+ // Open the compressed file for decompression.
+ //
+ // We do not use libarchive to open the file to insure we have proper
+ // Unicode and long path handling on both Linux and Windows.
+ Try<int_fd> fd = os::open(source, O_RDONLY | O_CLOEXEC, 0);
+
+ if (fd.isError()) {
+ return Error("Failed to open file '" + source + "': " + fd.error());
+ }
+
+#ifdef __WINDOWS__
+ int fd_real = fd.get().crt();
+#else
+ int fd_real = fd.get();
+#endif // __WINDOWS__
+
+ int result = archive_read_open_fd(
+ reader.get(),
+ fd_real,
+ 10240);
+ if (result) {
+ os::close(fd.get());
+ return Error(archive_error_string(reader.get()));
+ }
+
+ // Loop through file headers in the archive stream.
+ while (true) {
+ // Read the next header from the input stream.
+ struct archive_entry *entry;
+ result = archive_read_next_header(reader.get(), &entry);
+ if (result == ARCHIVE_EOF) {
+ break;
+ }
+ if (handleError(result, reader.get(), returnStatus)) {
+ os::close(fd.get());
+ return returnStatus;
+ }
+
+ // If a destination path is specified, update the entry to reflect it.
+ // We assume the destination directory already exists.
+
+ if (!destination.empty()) {
+ std::string path = path::join(destination,
+ archive_entry_pathname(entry));
+ archive_entry_update_pathname_utf8(entry, path.c_str());
+ }
+
+ result = archive_write_header(writer.get(), entry);
+ if (handleError(result, writer.get(), returnStatus)) {
+ os::close(fd.get());
+ return returnStatus;
+ }
+
+ if (archive_entry_size(entry) > 0) {
+ const void *buff;
+ size_t size;
+#if ARCHIVE_VERSION_NUMBER >= 3000000
+ int64_t offset;
+#else
+ off_t offset;
+#endif
+
+ // Loop through file data blocks until end of file.
+ while (true) {
+ result = archive_read_data_block(
+ reader.get(),
+ &buff,
+ &size,
+ &offset);
+ if (result == ARCHIVE_EOF) {
+ break;
+ }
+ if (handleError(result, reader.get(), returnStatus)) {
+ break;
+ }
+
+ result = archive_write_data_block(writer.get(), buff, size, offset);
+ if (handleError(result, writer.get(), returnStatus)) {
+ break;
+ }
+ }
+
+ // If an error happened dealing with file contents, abort processing.
+ if (result < ARCHIVE_WARN) {
+ break;
+ }
+ }
+
+ result = archive_write_finish_entry(writer.get());
+ if (handleError(result, writer.get(), returnStatus)) {
+ break;
+ }
+ }
+
+ os::close(fd.get());
+ return returnStatus;
+}
+
+} // namespace archiver {
+
+#endif // __STOUT_ARCHIVER_HPP__
diff --git a/3rdparty/stout/tests/CMakeLists.txt b/3rdparty/stout/tests/CMakeLists.txt
index 43c5e873a3..8e4b2cb140 100644
--- a/3rdparty/stout/tests/CMakeLists.txt
+++ b/3rdparty/stout/tests/CMakeLists.txt
@@ -17,6 +17,7 @@
# STOUT TESTS.
##############
set(STOUT_ROOT_TESTS_SRC
+ archiver_tests.cpp
base64_tests.cpp
bits_tests.cpp
bytes_tests.cpp
@@ -102,7 +103,9 @@ add_custom_command(
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/protobuf_tests.proto)
add_executable(stout-tests EXCLUDE_FROM_ALL ${STOUT_TESTS_SRC})
-target_link_libraries(stout-tests PRIVATE stout googletest)
+add_dependencies(stout-tests libarchive)
+
+target_link_libraries(stout-tests PRIVATE stout googletest libarchive)
target_include_directories(stout-tests PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
# ADD TEST TARGET (runs when you do, e.g., `make check`).
diff --git a/3rdparty/stout/tests/archiver_tests.cpp b/3rdparty/stout/tests/archiver_tests.cpp
new file mode 100644
index 0000000000..09e5d2482e
--- /dev/null
+++ b/3rdparty/stout/tests/archiver_tests.cpp
@@ -0,0 +1,454 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+#include <string>
+
+#ifdef __WINDOWS__
+#include <stout/windows.hpp>
+#endif
+
+#include <stout/archiver.hpp>
+#include <stout/base64.hpp>
+#include <stout/gtest.hpp>
+#include <stout/os.hpp>
+#include <stout/os/write.hpp>
+
+#include <stout/tests/utils.hpp>
+
+using std::string;
+
+
+class ArchiverTest : public TemporaryDirectoryTest {};
+
+// No input file should return some error, not read from stdin.
+TEST_F(ArchiverTest, ExtractEmptyInputFile)
+{
+ EXPECT_ERROR(archiver::extract("", ""));
+}
+
+// File that does not exist should return some error.
+TEST_F(ArchiverTest, ExtractInputFileNotFound)
+{
+ // Construct a temporary file path that is guarenteed unique.
+ Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
+ ASSERT_SOME(dir);
+
+ string path = path::join(dir.get(), "ThisFileDoesNotExist");
+
+ EXPECT_ERROR(archiver::extract(path, ""));
+}
+
+TEST_F(ArchiverTest, ExtractTarGzFile)
+{
+ // Construct a hello.tar.gz file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 22 22 2018-02-21 10:06 hello Howdy there, partner!\n
+ // -------- ------- ------- ------
+ // 22 22 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "H4sICE61jVoAA2hlbGxvLnRhcgDtzjEOwjAQRNGtOcXSU9hx7FyBa0RgK0IR"
+ "RsYIcfsEaGhQqggh/VfsFLPFDHEcs6zLzEJon2k7bz7zrQliXdM6Nx/vxVgb"
+ "uiBqVt71crvWvqjKKaZ0yOnr31L/p/b5fnxoHWKJO730pZ5j2W5+vQoAAAAA"
+ "AAAAAAAAAAAAsGQC2DPIjgAoAAA=").get()));
+
+ // Note: The file does NOT have a .tar.gz extension. We could rename
+ // it, but libarchive doesn't care about extensions. It determines
+ // the format from the contents of the file. So this is tested here
+ // as well.
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Howdy there, partner!\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractTarFile)
+{
+ // Construct a hello.tar file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // The .tar file, since not compressed, is long. So go through some
+ // pains to construct the contents fo the file programmatically.
+ //
+ // We could skip the .tar file test, but it's worth having it.
+
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 10240 10240 2018-02-21 10:06 hello Howdy there, partner (.tar)!\n
+ // -------- ------- ------- ------
+ // 10240 10240 1 file
+
+ string tarContents =
+ "aGVsbG8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAADAwMDA2NjQAMDAwMTc1MAAwMDAxNzUwADAwMDAwMDAwMDM1"
+ "ADEzMjQ1MTA2NTE1ADAxMTY3NAAgMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB1c3RhciAgAGplZmZj"
+ "b2YAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAamVmZmNvZgAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+ "AAAAAAAAAAAAAAAAAAAAAABIb3dkeSB0aGVyZSwgcGFydG5lciEgKC50YXIp"
+ "CgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+
+ for (int i = 0; i < 214; i++)
+ {
+ tarContents +=
+ "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+ }
+
+ tarContents += "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==";
+
+ // Now write out the .tar file, extract contents, and verify results
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(tarContents).get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Howdy there, partner! (.tar)\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractZipFile)
+{
+ // Construct a hello.zip file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 189 189 2018-02-26 15:06 hello Howdy there, partner! (.zip)\n
+ // -------- ------- ------- ------
+ // 189 189 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "UEsDBAoAAAAAAMZ4WkxFOXeVHQAAAB0AAAAFABwAaGVsbG9VVAkAA+SSlFrk"
+ "kpRadXgLAAEE6AMAAAToAwAASG93ZHkgdGhlcmUsIHBhcnRuZXIhICguemlw"
+ "KQpQSwECHgMKAAAAAADGeFpMRTl3lR0AAAAdAAAABQAYAAAAAAABAAAAtIEA"
+ "AAAAaGVsbG9VVAUAA+SSlFp1eAsAAQToAwAABOgDAABQSwUGAAAAAAEAAQBL"
+ "AAAAXAAAAAAA").get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Howdy there, partner! (.zip)\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractInvalidZipFile)
+{
+ // Construct a hello.zip file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Write broken zip to file [bad CRC 440a6aa5 (should be af083b2d)].
+ // Length Date Time CRC expected CRC actual Name Content
+ // -------- ---------- ----- ------------ ---------- ---- ------
+ // 12 2016-03-19 10:08 af083b2d 440a6aa5 world hello hello\n
+ // -------- ------- ------
+ // 12 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "UEsDBAoAAAAAABBRc0gtOwivDAAAAAwAAAAFABwAd29ybG9VVAkAAxAX7VYQ"
+ "F+1WdXgLAAEE6AMAAARkAAAAaGVsbG8gaGVsbG8KUEsBAh4DCgAAAAAAEFFz"
+ "SC07CK8MAAAADAAAAAUAGAAAAAAAAQAAAKSBAAAAAHdvcmxkVVQFAAMQF+1W"
+ "dXgLAAEE6AMAAARkAAAAUEsFBgAAAAABAAEASwAAAEsAAAAAAA==").get()));
+
+ EXPECT_ERROR(archiver::extract(path.get(), ""));
+}
+
+
+TEST_F(ArchiverTest, ExtractZipFileWithDuplicatedEntries)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create zip file with duplicates.
+ // Length Method Size Cmpr Date Time CRC-32 Name Content
+ // -------- ------ ------- ---- ---------- ----- -------- ---- -------
+ // 1 Stored 1 0% 2016-03-18 22:49 83dcefb7 A 1
+ // 1 Stored 1 0% 2016-03-18 22:49 1ad5be0d A 2
+ // -------- ------- --- ------- -------
+ // 2 2 0% 2 files
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "UEsDBBQAAAAAADC2cki379yDAQAAAAEAAAABAAAAQTFQSwMEFAAAAAAAMrZy"
+ "SA2+1RoBAAAAAQAAAAEAAABBMlBLAQIUAxQAAAAAADC2cki379yDAQAAAAEA"
+ "AAABAAAAAAAAAAAAAACAAQAAAABBUEsBAhQDFAAAAAAAMrZySA2+1RoBAAAA"
+ "AQAAAAEAAAAAAAAAAAAAAIABIAAAAEFQSwUGAAAAAAIAAgBeAAAAQAAAAAAA").get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "A");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("2", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractTarXZFile)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create an tar.xz compressed file
+ //
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 192 192 2018-02-27 15:34 hello Hello world (xz)\n
+ // -------- ------- ------- ------
+ // 192 192 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "/Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4Cf/AH5dADQZSe6N1/i4P8k3jGgA"
+ "rB4mJjQrf8ka7ajHWIxeYZoS+eGuA0Br4ooXZVdW4dnh8GpgDlbdfMQrOOPA"
+ "aJE3B9L56mP0ThtjwNuMhhc8/xiXsFOVeUf/xbgcqognut0NZNetr0p+FA/O"
+ "K6NqFHAjzSaANcbNj+iFfqY3sC/mAAAAADpda78LIiMIAAGaAYBQAADDUC3D"
+ "scRn+wIAAAAABFla").get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Hello world (xz)\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractTarBZ2File)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create an tar.bz2 compressed file
+ //
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 148 148 2018-02-27 15:34 hello Hello world (bzip2)\n
+ // -------- ------- ------- ------
+ // 148 148 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "QlpoOTFBWSZTWZo+haYAAH//hMIRAgBAYH+AAEAACH903pAABAAIIAB0EpEa"
+ "IeiMJtAIeRP1BlNCA00AAAA+x2lRZBAgaACRM0TvUjA5RJAR6BfGS3MjVUIh"
+ "IUI0Yww9tmran651Du0Hk5ZN4pbSxgs5xlAlIjtgOImyv+auHhIXnipV/xXy"
+ "iIHQu5IpwoSE0fQtMA==").get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Hello world (bzip2)\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractTarBz2GzFile)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create an tar.bz2.gz compressed file
+ //
+ // Verify that archives compressed twice (in this case, .bzip2.gz)
+ // work. libarchive will keep processing until fully extracted.
+ //
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 185 185 2018-02-27 15:46 hello Hello world (bzip2)\n
+ // -------- ------- ------- ------
+ // 185 185 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "H4sICOPtlVoAA2hlbGxvLnRhci5iejIAAZQAa/9CWmg5MUFZJlNZmj6FpgAA"
+ "f/+EwhECAEBgf4AAQAAIf3TekAAEAAggAHQSkRoh6Iwm0Ah5E/UGU0IDTQAA"
+ "AD7HaVFkECBoAJEzRO9SMDlEkBHoF8ZLcyNVQiEhQjRjDD22atqfrnUO7QeT"
+ "lk3iltLGCznGUCUiO2A4ibK/5q4eEheeKlX/FfKIgdC7kinChITR9C0wSQeY"
+ "TJQAAAA=").get()));
+
+ EXPECT_SOME(archiver::extract(path.get(), ""));
+
+ string extractedFile = path::join(os::getcwd(), "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Hello world (bzip2)\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractBz2FileFails)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create an .bz2 compressed file
+ //
+ // libarchive does not appear to work without some sort of container
+ // (tar or zip or whatever). Verify that a regular file, compressed,
+ // will fail.
+ //
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 63 63 2018-02-27 17:00 hello Hello world (bzip2)\n
+ // -------- ------- ------- ------
+ // 63 63 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "QlpoOTFBWSZTWTMaBKkAAANdgAAQQGAQAABAFiTQkCAAIhGCD1HoUwAE0auv"
+ "Imhs/86EgGxdyRThQkDMaBKk").get()));
+
+ EXPECT_ERROR(archiver::extract(path.get(), ""));
+}
+
+
+TEST_F(ArchiverTest, ExtractGzFileFails)
+{
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> path = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(path);
+
+ // Create an .gz compressed file
+ //
+ // libarchive does not appear to work without some sort of container
+ // (tar or zip or whatever). Verify that a regular file, compressed,
+ // will fail.
+ //
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 43 43 2018-03-21 16:59 hello Hello world (gz)\n
+ // -------- ------- ------- ------
+ // 43 43 1 file
+
+ ASSERT_SOME(os::write(path.get(), base64::decode(
+ "H4sICNjxsloAA2hlbGxvAPNIzcnJVyjPL8pJUdBIr9LkAgAwtvTdEQAAAA==").get()));
+
+ EXPECT_ERROR(archiver::extract(path.get(), ""));
+}
+
+
+TEST_F(ArchiverTest, ExtractTarGzFileWithDestinationDir)
+{
+ // Construct a hello.tar.gz file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> sourcePath = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(sourcePath);
+
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 22 22 2018-02-21 10:06 hello Howdy there, partner!\n
+ // -------- ------- ------- ------
+ // 22 22 1 file
+
+ ASSERT_SOME(os::write(sourcePath.get(), base64::decode(
+ "H4sICE61jVoAA2hlbGxvLnRhcgDtzjEOwjAQRNGtOcXSU9hx7FyBa0RgK0IR"
+ "RsYIcfsEaGhQqggh/VfsFLPFDHEcs6zLzEJon2k7bz7zrQliXdM6Nx/vxVgb"
+ "uiBqVt71crvWvqjKKaZ0yOnr31L/p/b5fnxoHWKJO730pZ5j2W5+vQoAAAAA"
+ "AAAAAAAAAAAAsGQC2DPIjgAoAAA=").get()));
+
+ // Make a destination directory to extract the archive to
+ string destDir = path::join(dir, "somedestination");
+ ASSERT_SOME(os::mkdir(destDir));
+
+ // Note: The file does NOT have a .tar.gz extension. We could rename
+ // it, but libarchive doesn't care about extensions. It determines
+ // the format from the contents of the file. So this is tested here
+ // as well.
+ //
+ // Note: In this test, we extrat the file to a destination directory
+ // and expect to find it there.
+ EXPECT_SOME(archiver::extract(sourcePath.get(), destDir));
+
+ string extractedFile = path::join(destDir, "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Howdy there, partner!\n", os::read(extractedFile));
+}
+
+
+TEST_F(ArchiverTest, ExtractZipFileWithDestinationDir)
+{
+ // Construct a hello.zip file that can be extracted
+ string dir = path::join(os::getcwd(), "somedir");
+ ASSERT_SOME(os::mkdir(dir));
+
+ Try<string> sourcePath = os::mktemp(path::join(dir, "XXXXXX"));
+ ASSERT_SOME(sourcePath);
+
+ // Length Size Date Time Name Content
+ // -------- ------- ---------- ----- ---- ------
+ // 189 189 2018-02-26 15:06 hello Howdy there, partner! (.zip)\n
+ // -------- ------- ------- ------
+ // 189 189 1 file
+
+ ASSERT_SOME(os::write(sourcePath.get(), base64::decode(
+ "UEsDBAoAAAAAAMZ4WkxFOXeVHQAAAB0AAAAFABwAaGVsbG9VVAkAA+SSlFrk"
+ "kpRadXgLAAEE6AMAAAToAwAASG93ZHkgdGhlcmUsIHBhcnRuZXIhICguemlw"
+ "KQpQSwECHgMKAAAAAADGeFpMRTl3lR0AAAAdAAAABQAYAAAAAAABAAAAtIEA"
+ "AAAAaGVsbG9VVAUAA+SSlFp1eAsAAQToAwAABOgDAABQSwUGAAAAAAEAAQBL"
+ "AAAAXAAAAAAA").get()));
+
+ // Make a destination directory to extract the archive to
+ string destDir = path::join(dir, "somedestination");
+ ASSERT_SOME(os::mkdir(destDir));
+
+ EXPECT_SOME(archiver::extract(sourcePath.get(), destDir));
+
+ string extractedFile = path::join(destDir, "hello");
+ ASSERT_TRUE(os::exists(extractedFile));
+
+ ASSERT_SOME_EQ("Howdy there, partner! (.zip)\n", os::read(extractedFile));
+}
diff --git a/3rdparty/versions.am b/3rdparty/versions.am
index ada998d62d..57792800c4 100644
--- a/3rdparty/versions.am
+++ b/3rdparty/versions.am
@@ -30,6 +30,7 @@ GRPC_VERSION = 1.10.0
HTTP_PARSER_VERSION = 2.6.2
JEMALLOC_VERSION = 5.0.1
LEVELDB_VERSION = 1.19
+LIBARCHIVE_VERSION = 3.3.2
LIBEV_VERSION = 4.22
NVML_VERSION = 352.79
PICOJSON_VERSION = 1.3.0
diff --git a/3rdparty/xz-5.2.3.patch b/3rdparty/xz-5.2.3.patch
new file mode 100644
index 0000000000..70f1cb9a26
--- /dev/null
+++ b/3rdparty/xz-5.2.3.patch
@@ -0,0 +1,187 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+new file mode 100644
+index 0000000..fbf90f0
+--- /dev/null
++++ b/CMakeLists.txt
+@@ -0,0 +1,181 @@
++project(xz C)
++
++add_library(lzma
++ src/common/tuklib_cpucores.c
++ src/common/tuklib_physmem.c
++ src/liblzma/check/check.c
++ src/liblzma/check/crc32_fast.c
++ src/liblzma/check/crc32_table.c
++ src/liblzma/check/crc64_fast.c
++ src/liblzma/check/crc64_table.c
++ src/liblzma/check/sha256.c
++ src/liblzma/common/alone_decoder.c
++ src/liblzma/common/alone_encoder.c
++ src/liblzma/common/auto_decoder.c
++ src/liblzma/common/block_buffer_decoder.c
++ src/liblzma/common/block_buffer_encoder.c
++ src/liblzma/common/block_decoder.c
++ src/liblzma/common/block_encoder.c
++ src/liblzma/common/block_header_decoder.c
++ src/liblzma/common/block_header_encoder.c
++ src/liblzma/common/block_util.c
++ src/liblzma/common/common.c
++ src/liblzma/common/easy_buffer_encoder.c
++ src/liblzma/common/easy_decoder_memusage.c
++ src/liblzma/common/easy_encoder.c
++ src/liblzma/common/easy_encoder_memusage.c
++ src/liblzma/common/easy_preset.c
++ src/liblzma/common/filter_buffer_decoder.c
++ src/liblzma/common/filter_buffer_encoder.c
++ src/liblzma/common/filter_common.c
++ src/liblzma/common/filter_decoder.c
++ src/liblzma/common/filter_encoder.c
++ src/liblzma/common/filter_flags_decoder.c
++ src/liblzma/common/filter_flags_encoder.c
++ src/liblzma/common/hardware_cputhreads.c
++ src/liblzma/common/hardware_physmem.c
++ src/liblzma/common/index.c
++ src/liblzma/common/index_decoder.c
++ src/liblzma/common/index_encoder.c
++ src/liblzma/common/index_hash.c
++ src/liblzma/common/outqueue.c
++ src/liblzma/common/stream_buffer_decoder.c
++ src/liblzma/common/stream_buffer_encoder.c
++ src/liblzma/common/stream_decoder.c
++ src/liblzma/common/stream_encoder.c
++ src/liblzma/common/stream_encoder_mt.c
++ src/liblzma/common/stream_flags_common.c
++ src/liblzma/common/stream_flags_decoder.c
++ src/liblzma/common/stream_flags_encoder.c
++ src/liblzma/common/vli_decoder.c
++ src/liblzma/common/vli_encoder.c
++ src/liblzma/common/vli_size.c
++ src/liblzma/delta/delta_common.c
++ src/liblzma/delta/delta_decoder.c
++ src/liblzma/delta/delta_encoder.c
++ src/liblzma/lzma/fastpos_table.c
++ src/liblzma/lzma/lzma2_decoder.c
++ src/liblzma/lzma/lzma2_encoder.c
++ src/liblzma/lzma/lzma_decoder.c
++ src/liblzma/lzma/lzma_encoder.c
++ src/liblzma/lzma/lzma_encoder_optimum_fast.c
++ src/liblzma/lzma/lzma_encoder_optimum_normal.c
++ src/liblzma/lzma/lzma_encoder_presets.c
++ src/liblzma/lz/lz_decoder.c
++ src/liblzma/lz/lz_encoder.c
++ src/liblzma/lz/lz_encoder_mf.c
++ src/liblzma/rangecoder/price_table.c
++ src/liblzma/simple/arm.c
++ src/liblzma/simple/armthumb.c
++ src/liblzma/simple/ia64.c
++ src/liblzma/simple/powerpc.c
++ src/liblzma/simple/simple_coder.c
++ src/liblzma/simple/simple_decoder.c
++ src/liblzma/simple/simple_encoder.c
++ src/liblzma/simple/sparc.c
++ src/liblzma/simple/x86.c
++ src/common/mythread.h
++ src/common/sysdefs.h
++ src/common/tuklib_common.h
++ src/common/tuklib_config.h
++ src/common/tuklib_cpucores.h
++ src/common/tuklib_integer.h
++ src/common/tuklib_physmem.h
++ src/liblzma/api/lzma.h
++ src/liblzma/api/lzma/base.h
++ src/liblzma/api/lzma/bcj.h
++ src/liblzma/api/lzma/block.h
++ src/liblzma/api/lzma/check.h
++ src/liblzma/api/lzma/container.h
++ src/liblzma/api/lzma/delta.h
++ src/liblzma/api/lzma/filter.h
++ src/liblzma/api/lzma/hardware.h
++ src/liblzma/api/lzma/index.h
++ src/liblzma/api/lzma/index_hash.h
++ src/liblzma/api/lzma/lzma12.h
++ src/liblzma/api/lzma/stream_flags.h
++ src/liblzma/api/lzma/version.h
++ src/liblzma/api/lzma/vli.h
++ src/liblzma/check/check.h
++ src/liblzma/check/crc32_table_be.h
++ src/liblzma/check/crc32_table_le.h
++ src/liblzma/check/crc64_table_be.h
++ src/liblzma/check/crc64_table_le.h
++ src/liblzma/check/crc_macros.h
++ src/liblzma/common/alone_decoder.h
++ src/liblzma/common/block_buffer_encoder.h
++ src/liblzma/common/block_decoder.h
++ src/liblzma/common/block_encoder.h
++ src/liblzma/common/common.h
++ src/liblzma/common/easy_preset.h
++ src/liblzma/common/filter_common.h
++ src/liblzma/common/filter_decoder.h
++ src/liblzma/common/filter_encoder.h
++ src/liblzma/common/index.h
++ src/liblzma/common/index_encoder.h
++ src/liblzma/common/memcmplen.h
++ src/liblzma/common/outqueue.h
++ src/liblzma/common/stream_decoder.h
++ src/liblzma/common/stream_flags_common.h
++ src/liblzma/delta/delta_common.h
++ src/liblzma/delta/delta_decoder.h
++ src/liblzma/delta/delta_encoder.h
++ src/liblzma/delta/delta_private.h
++ src/liblzma/lzma/fastpos.h
++ src/liblzma/lzma/lzma2_decoder.h
++ src/liblzma/lzma/lzma2_encoder.h
++ src/liblzma/lzma/lzma_common.h
++ src/liblzma/lzma/lzma_decoder.h
++ src/liblzma/lzma/lzma_encoder.h
++ src/liblzma/lzma/lzma_encoder_private.h
++ src/liblzma/lz/lz_decoder.h
++ src/liblzma/lz/lz_encoder.h
++ src/liblzma/lz/lz_encoder_hash.h
++ src/liblzma/lz/lz_encoder_hash_table.h
++ src/liblzma/rangecoder/price.h
++ src/liblzma/rangecoder/range_common.h
++ src/liblzma/rangecoder/range_decoder.h
++ src/liblzma/rangecoder/range_encoder.h
++ src/liblzma/simple/simple_coder.h
++ src/liblzma/simple/simple_decoder.h
++ src/liblzma/simple/simple_encoder.h
++ src/liblzma/simple/simple_private.h
++ windows/config.h
++)
++
++target_include_directories(lzma
++ PRIVATE
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/api>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/check>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/common>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/lzma>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/lz>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/simple>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/delta>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/rangecoder>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/common>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/common>
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/windows>
++ INTERFACE
++ $<INSTALL_INTERFACE:include>
++ PUBLIC
++ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/liblzma/api>
++)
++
++target_compile_definitions(lzma
++ PRIVATE
++ _CRT_SECURE_NO_WARNINGS
++ _CRT_STDC_NO_DEPRECATE
++ HAVE_CONFIG_H
++)
++
++install(TARGETS lzma
++ DESTINATION ${XZ_INSTALL_DIR})
++
++install(FILES src/liblzma/api/lzma.h
++ DESTINATION ${XZ_INSTALL_DIR})
++
++install(DIRECTORY src/liblzma/api/lzma
++ DESTINATION ${XZ_INSTALL_DIR})
diff --git a/3rdparty/xz-5.2.3.tar.gz b/3rdparty/xz-5.2.3.tar.gz
new file mode 100644
index 0000000000..147ce20275
Binary files /dev/null and b/3rdparty/xz-5.2.3.tar.gz differ
diff --git a/configure.ac b/configure.ac
index 6d9c8ceb08..e2b6e1e97d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -408,6 +408,11 @@ AC_ARG_WITH([libevent],
[specify where to locate the libevent library]),
[], [])
+AC_ARG_WITH([libarchive],
+ AS_HELP_STRING([--with-libarchive=@<:@=DIR@:>@],
+ [specify where to locate the libarchive library]),
+ [without_bundled_libarchive=yes], [])
+
AC_ARG_WITH([libprocess],
AS_HELP_STRING([--with-libprocess=@<:@=DIR@:>@],
[specify where to locate the libprocess library]),
@@ -1453,6 +1458,41 @@ fi
AM_CONDITIONAL([ENABLE_LIBEVENT], [test x"$enable_libevent" = "xyes"])
+# Check if user has asked us to use a preinstalled libarchive, or if
+# they asked us to ignore all bundled libraries while compiling and
+# linking.
+if test -n "`echo $with_libarchive`"; then
+ CPPFLAGS="$CPPFLAGS -I${with_libarchive}/include"
+ LDFLAGS="$LDFLAGS -L${with_libarchive}/lib"
+fi
+
+if test "x$without_bundled_libarchive" = "xyes"; then
+ # Check if headers and library were located.
+ AC_CHECK_HEADERS([archive.h], [found_libarchive=yes])
+
+ if test "x$found_libarchive" = "xyes"; then
+ with_bundled_libarchive=no
+ else
+ AC_MSG_ERROR([cannot find libarchive
+-------------------------------------------------------------------
+You have requested the use of a non-bundled libarchive but no suitable
+libarchive could be found.
+
+You may want specify the location of libarchive by providing a prefix
+path via --with-libarchive=DIR, or check that the path you provided is
+correct if you're already doing this.
+-------------------------------------------------------------------
+])
+ fi
+
+else
+ with_bundled_libarchive=yes
+fi
+
+AM_CONDITIONAL([WITH_BUNDLED_LIBARCHIVE], [test "x$with_bundled_libarchive" = "xyes"])
+
+
+
# Check if user has asked us to use a preinstalled libprocess, or if
# they asked us to ignore all bundled libraries while compiling and
# linking.
diff --git a/src/Makefile.am b/src/Makefile.am
index c08ac6e2f5..19dba3b6bf 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -35,6 +35,7 @@ GRPC = 3rdparty/grpc-$(GRPC_VERSION)
GTEST = $(GOOGLETEST)/googletest
JEMALLOC = 3rdparty/jemalloc-$(JEMALLOC_VERSION)
LEVELDB = 3rdparty/leveldb-$(LEVELDB_VERSION)
+LIBARCHIVE = 3rdparty/libarchive-$(LIBARCHIVE_VERSION)
LIBPROCESS = 3rdparty/libprocess
NVML = 3rdparty/nvml-$(NVML_VERSION)
PICOJSON = 3rdparty/picojson-$(PICOJSON_VERSION)
@@ -211,13 +212,22 @@ LIB_LEVELDB = -lleveldb
LDADD += -lleveldb
endif
-if WITH_BUNDLED_LIBPROCESS
-MESOS_CPPFLAGS += -I$(top_srcdir)/$(LIBPROCESS)/include
-LIB_PROCESS = ../$(LIBPROCESS)/libprocess.la
+if WITH_BUNDLED_LIBARCHIVE
+MESOS_CPPFLAGS += -I../$(LIBARCHIVE)/libarchive/
+LIB_LIBARCHIVE = ../$(LIBARCHIVE)/.libs/libarchive.la
+LDADD += ../$(LIBARCHIVE)/.libs/libarchive.la
else
-LIB_PROCESS = -lprocess
+LIB_LIBARCHIVE = -larchive
+LDADD += -larchive
endif
+#if WITH_BUNDLED_LIBPROCESS
+MESOS_CPPFLAGS += -I$(top_srcdir)/$(LIBPROCESS)/include
+LIB_PROCESS = ../$(LIBPROCESS)/libprocess.la
+#else
+#LIB_PROCESS = -lprocess
+#endif
+
if WITH_BUNDLED_NVML
MESOS_CPPFLAGS += -I../$(NVML)
endif
@@ -1663,6 +1673,7 @@ libmesos_la_LIBADD = \
$(LIB_GLOG) \
$(LIB_GRPC) \
$(LIB_LEVELDB) \
+ $(LIB_LIBARCHIVE) \
$(LIB_PROCESS) \
$(LIB_PROTOBUF) \
$(LIB_ZOOKEEPER) \
diff --git a/src/launcher/fetcher.cpp b/src/launcher/fetcher.cpp
index 4cff7273fd..de4bdf9247 100644
--- a/src/launcher/fetcher.cpp
+++ b/src/launcher/fetcher.cpp
@@ -20,6 +20,7 @@
#include <process/owned.hpp>
#include <process/subprocess.hpp>
+#include <stout/archiver.hpp>
#include <stout/json.hpp>
#include <stout/net.hpp>
#include <stout/option.hpp>
@@ -80,9 +81,19 @@ static Try<bool> extract(
strings::endsWith(sourcePath, ".tbz2") ||
strings::endsWith(sourcePath, ".tar.bz2") ||
strings::endsWith(sourcePath, ".txz") ||
- strings::endsWith(sourcePath, ".tar.xz")) {
- command = {"tar", "-C", destinationDirectory, "-xf", sourcePath};
+ strings::endsWith(sourcePath, ".tar.xz") ||
+ strings::endsWith(sourcePath, ".zip")) {
+ Try<Nothing> result = archiver::extract(sourcePath, destinationDirectory);
+ if (result.isError()) {
+ return Error(
+ "Failed to extract archive '" + sourcePath +
+ "' to '" + destinationDirectory + "': " + result.error());
+ }
+ return true;
} else if (strings::endsWith(sourcePath, ".gz")) {
+ // Unfortunately, libarchive can't extract bare files, so leave this to
+ // the 'gunzip' program, if it exists.
+
string pathWithoutExtension = sourcePath.substr(0, sourcePath.length() - 3);
string filename = Path(pathWithoutExtension).basename();
string destinationPath = path::join(destinationDirectory, filename);
@@ -90,20 +101,6 @@ static Try<bool> extract(
command = {"gunzip", "-d", "-c"};
in = Subprocess::PATH(sourcePath);
out = Subprocess::PATH(destinationPath);
- } else if (strings::endsWith(sourcePath, ".zip")) {
-#ifdef __WINDOWS__
- command = {"powershell",
- "-NoProfile",
- "-Command",
- "Expand-Archive",
- "-Force",
- "-Path",
- sourcePath,
- "-DestinationPath",
- destinationDirectory};
-#else
- command = {"unzip", "-o", "-d", destinationDirectory, sourcePath};
-#endif // __WINDOWS__
} else {
return false;
}
diff --git a/src/tests/fetcher_tests.cpp b/src/tests/fetcher_tests.cpp
index 311be6a36f..077f68dfcb 100644
--- a/src/tests/fetcher_tests.cpp
+++ b/src/tests/fetcher_tests.cpp
@@ -1008,33 +1008,17 @@ TEST_F(FetcherTest, UNZIP_ExtractInvalidFile)
os::getcwd(),
None());
-#ifdef __WINDOWS__
- // On Windows, PowerShell doesn't consider a CRC error to be an error,
- // so it succeeds, whereas the zip utility errors.
- //
- // TODO(coffler): When we move to programmatically dealing with various
- // data files (tar, gzip, zip, etc), we should be able to resolve this.
- // See MESOS-7740 for further details.
- AWAIT_READY(fetch);
-#else
AWAIT_FAILED(fetch);
-#endif // __WINDOWS__
- string extractedFile = path::join(os::getcwd(), "world");
- ASSERT_TRUE(os::exists(extractedFile));
+ // libarchive is different than zip:
+ //
+ // Zip, upon CRC error, will still extract the file (and return an error).
+ // libarchive, on CRC error, will abort (and not create the file).
- ASSERT_SOME_EQ("hello hello\n", os::read(extractedFile));
+ string extractedFile = path::join(os::getcwd(), "world");
+ ASSERT_FALSE(os::exists(extractedFile));
-#ifdef __WINDOWS__
- // TODO(coffler): Eliminate with programmatic decoding of container files.
- // See MESOS-7740 for further details.
- //
- // On Windows, PowerShell doesn't consider a CRC error to be an error.
- // Adjust metrics appropriately to not expect an error back.
- verifyMetrics(1, 0);
-#else
verifyMetrics(0, 1);
-#endif // __WINDOWS__
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services