You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2019/07/08 16:44:24 UTC
[mesos] branch master updated: Windows: Removed multiple categories
of sources from the build.
This is an automated email from the ASF dual-hosted git repository.
josephwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
The following commit(s) were added to refs/heads/master by this push:
new 6a026e3 Windows: Removed multiple categories of sources from the build.
6a026e3 is described below
commit 6a026e3fadfccc821d83ab030d318828af2c94a7
Author: Joseph Wu <jo...@apache.org>
AuthorDate: Fri May 24 13:45:31 2019 -0700
Windows: Removed multiple categories of sources from the build.
This removes some categories of sources from the Windows build,
where it is possible to do so with minimal ifdef-ing.
The features removed are all Linux-specific features that cannot be
feasibly ported to Windows, including:
* Container Storage Interface (CSI)
* Docker image provisioning, specifically related to V2
* Open Container Interface
* Volume GID Manager
Protobufs are excluded where possible, but many of the above categories
of protobufs are interleaved with other protobufs or source code,
which makes exclusion non-trivial. For example, libseccomp is a
Linux-only feature, but its protobufs are now required to build the
Mesos containerizer's protobufs.
Docker image provisioning was semi-trivial to exclude, because the
related components (provisioner & URI fetcher) are somewhat modularized.
Review: https://reviews.apache.org/r/70740
---
src/CMakeLists.txt | 95 +++++++++++++---------
src/slave/containerizer/mesos/containerizer.cpp | 1 -
.../containerizer/mesos/provisioner/store.cpp | 10 +++
src/tests/CMakeLists.txt | 8 +-
src/tests/uri_fetcher_tests.cpp | 8 ++
src/uri/fetcher.cpp | 2 +
src/uri/fetcher.hpp | 11 ++-
7 files changed, 89 insertions(+), 46 deletions(-)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 1d4f541..eb4e2ac 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -26,27 +26,24 @@ set(JAVA_PROTOBUF_SRC "")
#
# NOTE: The following `PROTOC_GENERATE` calls will list append to
# `PUBLIC_PROTOBUF_SRC`. The GRPC option is a noop if gRPC is disabled.
-PROTOC_GENERATE(GRPC LIB csi_v0 TARGET csi/v0/csi)
-PROTOC_GENERATE(GRPC LIB csi_v1 TARGET csi/v1/csi)
+
+if (NOT WIN32)
+ PROTOC_GENERATE(GRPC LIB csi_v0 TARGET csi/v0/csi)
+ PROTOC_GENERATE(GRPC LIB csi_v1 TARGET csi/v1/csi)
+endif ()
# NOTE: The following `PROTOC_GENERATE` calls will list append to
# `PUBLIC_PROTOBUF_SRC`.
PROTOC_GENERATE(TARGET mesos/agent/agent)
PROTOC_GENERATE(TARGET mesos/allocator/allocator)
-PROTOC_GENERATE(TARGET mesos/appc/spec)
PROTOC_GENERATE(TARGET mesos/authentication/authentication)
PROTOC_GENERATE(TARGET mesos/authorizer/acls)
PROTOC_GENERATE(TARGET mesos/authorizer/authorizer)
PROTOC_GENERATE(TARGET mesos/csi/types)
-PROTOC_GENERATE(TARGET mesos/docker/spec)
-PROTOC_GENERATE(TARGET mesos/docker/v1)
-PROTOC_GENERATE(TARGET mesos/docker/v2)
-PROTOC_GENERATE(TARGET mesos/docker/v2_2)
PROTOC_GENERATE(TARGET mesos/maintenance/maintenance)
PROTOC_GENERATE(TARGET mesos/master/master)
PROTOC_GENERATE(TARGET mesos/module/hook)
PROTOC_GENERATE(TARGET mesos/module/module)
-PROTOC_GENERATE(TARGET mesos/oci/spec)
PROTOC_GENERATE(TARGET mesos/quota/quota)
PROTOC_GENERATE(TARGET mesos/resource_provider/resource_provider)
PROTOC_GENERATE(TARGET mesos/seccomp/seccomp)
@@ -55,6 +52,18 @@ PROTOC_GENERATE(TARGET mesos/slave/oversubscription)
PROTOC_GENERATE(TARGET mesos/state/state)
PROTOC_GENERATE(TARGET mesos/uri/uri)
+# TODO(josephw): Remove or stub the provisioner in the Windows sources
+# so these can be excluded from the Windows build.
+PROTOC_GENERATE(TARGET mesos/appc/spec)
+PROTOC_GENERATE(TARGET mesos/docker/v1)
+
+if (NOT WIN32)
+ PROTOC_GENERATE(TARGET mesos/docker/spec)
+ PROTOC_GENERATE(TARGET mesos/docker/v2)
+ PROTOC_GENERATE(TARGET mesos/docker/v2_2)
+ PROTOC_GENERATE(TARGET mesos/oci/spec)
+endif ()
+
# NOTE: The following `PROTOC_GENERATE` calls will list append to
# `PUBLIC_PROTOBUF_SRC` and/or `JAVA_PROTOBUF_SRC`. The JAVA option is a noop if
# Java is disabled.
@@ -74,20 +83,23 @@ PROTOC_GENERATE(JAVA TARGET mesos/v1/scheduler/scheduler)
# NOTE: The following `PROTOC_GENERATE` calls will list append to
# `INTERNAL_PROTOBUF_SRC`.
+PROTOC_GENERATE(INTERNAL TARGET master/registry)
PROTOC_GENERATE(INTERNAL TARGET messages/flags)
PROTOC_GENERATE(INTERNAL TARGET messages/log)
PROTOC_GENERATE(INTERNAL TARGET messages/messages)
PROTOC_GENERATE(INTERNAL TARGET messages/state)
-PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/isolators/network/cni/spec)
-PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/isolators/docker/volume/state)
-PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/provisioner/docker/message)
-PROTOC_GENERATE(INTERNAL TARGET slave/state)
-PROTOC_GENERATE(INTERNAL TARGET slave/volume_gid_manager/state)
-PROTOC_GENERATE(INTERNAL TARGET master/registry)
PROTOC_GENERATE(INTERNAL TARGET resource_provider/registry)
PROTOC_GENERATE(INTERNAL TARGET resource_provider/state)
-PROTOC_GENERATE(INTERNAL TARGET resource_provider/storage/disk_profile)
-PROTOC_GENERATE(INTERNAL TARGET csi/state)
+PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/isolators/network/cni/spec)
+PROTOC_GENERATE(INTERNAL TARGET slave/state)
+
+if (NOT WIN32)
+ PROTOC_GENERATE(INTERNAL TARGET csi/state)
+ PROTOC_GENERATE(INTERNAL TARGET resource_provider/storage/disk_profile)
+ PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/isolators/docker/volume/state)
+ PROTOC_GENERATE(INTERNAL TARGET slave/containerizer/mesos/provisioner/docker/message)
+ PROTOC_GENERATE(INTERNAL TARGET slave/volume_gid_manager/state)
+endif ()
# BUILD PROTOBUFS.
@@ -163,35 +175,35 @@ set(AGENT_SRC
slave/containerizer/mesos/mount.cpp
slave/containerizer/mesos/paths.cpp
slave/containerizer/mesos/io/switchboard.cpp
+ slave/containerizer/mesos/isolators/environment_secret.cpp
+ slave/containerizer/mesos/isolators/filesystem/posix.cpp
slave/containerizer/mesos/provisioner/backend.cpp
slave/containerizer/mesos/provisioner/paths.cpp
slave/containerizer/mesos/provisioner/provisioner.cpp
slave/containerizer/mesos/provisioner/store.cpp
- slave/containerizer/mesos/provisioner/appc/cache.cpp
- slave/containerizer/mesos/provisioner/appc/fetcher.cpp
- slave/containerizer/mesos/provisioner/appc/paths.cpp
- slave/containerizer/mesos/provisioner/appc/store.cpp
slave/containerizer/mesos/provisioner/backends/copy.cpp
- slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
- slave/containerizer/mesos/provisioner/docker/metadata_manager.cpp
- slave/containerizer/mesos/provisioner/docker/paths.cpp
- slave/containerizer/mesos/provisioner/docker/puller.cpp
- slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
- slave/containerizer/mesos/provisioner/docker/store.cpp
slave/resource_estimators/noop.cpp)
if (NOT WIN32)
list(APPEND AGENT_SRC
slave/containerizer/mesos/utils.cpp
- slave/containerizer/mesos/isolators/environment_secret.cpp
slave/containerizer/mesos/isolators/docker/volume/driver.cpp
slave/containerizer/mesos/isolators/docker/volume/paths.cpp
- slave/containerizer/mesos/isolators/filesystem/posix.cpp
slave/containerizer/mesos/isolators/network/cni/paths.cpp
slave/containerizer/mesos/isolators/network/cni/spec.cpp
slave/containerizer/mesos/isolators/posix/disk.cpp
slave/containerizer/mesos/isolators/posix/rlimits.cpp
slave/containerizer/mesos/isolators/volume/sandbox_path.cpp
+ slave/containerizer/mesos/provisioner/appc/cache.cpp
+ slave/containerizer/mesos/provisioner/appc/fetcher.cpp
+ slave/containerizer/mesos/provisioner/appc/paths.cpp
+ slave/containerizer/mesos/provisioner/appc/store.cpp
+ slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
+ slave/containerizer/mesos/provisioner/docker/metadata_manager.cpp
+ slave/containerizer/mesos/provisioner/docker/paths.cpp
+ slave/containerizer/mesos/provisioner/docker/puller.cpp
+ slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
+ slave/containerizer/mesos/provisioner/docker/store.cpp
slave/containerizer/mesos/provisioner/utils.cpp
slave/volume_gid_manager/volume_gid_manager.cpp)
endif ()
@@ -432,13 +444,13 @@ set(RESOURCE_PROVIDER_SRC
resource_provider/manager.cpp
resource_provider/registrar.cpp
resource_provider/validation.cpp
- resource_provider/storage/disk_profile_adaptor.cpp
- resource_provider/storage/disk_profile_utils.cpp)
+ resource_provider/storage/disk_profile_adaptor.cpp)
# NOTE: The storage local resource provider uses Unix domain sockets to talk to
# CSI plugins, thus it is currently not supported on Windows.
if (NOT WIN32)
list(APPEND RESOURCE_PROVIDER_SRC
+ resource_provider/storage/disk_profile_utils.cpp
resource_provider/storage/provider.cpp)
endif ()
@@ -467,9 +479,13 @@ set(URI_SRC
uri/utils.cpp
uri/fetchers/copy.cpp
uri/fetchers/curl.cpp
- uri/fetchers/docker.cpp
uri/fetchers/hadoop.cpp)
+if (NOT WIN32)
+ list(APPEND URI_SRC
+ uri/fetchers/docker.cpp)
+endif ()
+
set(USAGE_SRC
usage/usage.cpp)
@@ -486,11 +502,11 @@ set(WATCHER_SRC
watcher/whitelist_watcher.cpp)
set(WIN32_SRC
- slave/containerizer/mesos/isolators/docker/runtime.cpp
+ # NOTE: Needed for a helper inside slave/paths.cpp.
+ csi/paths.cpp
+
slave/containerizer/mesos/isolators/windows/cpu.cpp
slave/containerizer/mesos/isolators/windows/mem.cpp
- slave/containerizer/mesos/isolators/environment_secret.cpp
- slave/containerizer/mesos/isolators/filesystem/posix.cpp
slave/containerizer/mesos/isolators/filesystem/windows.cpp)
set(ZOOKEEPER_SRC
@@ -502,11 +518,9 @@ set(ZOOKEEPER_SRC
set(MESOS_SRC
${AGENT_SRC}
- ${APPC_SRC}
${AUTHENTICATION_SRC}
${AUTHORIZER_SRC}
${COMMON_SRC}
- ${CSI_SRC}
${DOCKER_SRC}
${EXECUTOR_SRC}
${FILES_SRC}
@@ -518,7 +532,6 @@ set(MESOS_SRC
${MASTER_SRC}
${MESSAGES_SRC}
${MODULE_SRC}
- ${OCI_SRC}
${RESOURCE_PROVIDER_SRC}
${SCHEDULER_SRC}
${SECRET_SRC}
@@ -533,7 +546,10 @@ set(MESOS_SRC
if (NOT WIN32)
list(APPEND MESOS_SRC
+ ${APPC_SRC}
+ ${CSI_SRC}
${LOG_SRC}
+ ${OCI_SRC}
${POSIX_SRC})
endif ()
@@ -629,11 +645,14 @@ add_subdirectory(master)
add_subdirectory(python/cli_new)
add_subdirectory(python/executor)
add_subdirectory(python/scheduler)
-add_subdirectory(resource_provider/storage)
add_subdirectory(slave)
add_subdirectory(slave/containerizer/mesos)
add_subdirectory(usage)
+if (NOT WIN32)
+ add_subdirectory(resource_provider/storage)
+endif()
+
# BUILD THE MESOS TESTS.
########################
diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp
index 0432448..c9a369b 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -407,7 +407,6 @@ Try<MesosContainerizer*> MesosContainerizer::create(
#endif // __WINDOWS__
#ifdef __WINDOWS__
- {"docker/runtime", &DockerRuntimeIsolatorProcess::create},
{"windows/cpu", &WindowsCpuIsolatorProcess::create},
{"windows/mem", &WindowsMemIsolatorProcess::create},
#endif // __WINDOWS__
diff --git a/src/slave/containerizer/mesos/provisioner/store.cpp b/src/slave/containerizer/mesos/provisioner/store.cpp
index 11fce0e..5b03df0 100644
--- a/src/slave/containerizer/mesos/provisioner/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/store.cpp
@@ -27,9 +27,11 @@
#include "slave/containerizer/mesos/provisioner/store.hpp"
+#ifndef __WINDOWS__
#include "slave/containerizer/mesos/provisioner/appc/store.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
+#endif // __WINDOWS__
using std::string;
using std::vector;
@@ -44,6 +46,13 @@ Try<hashmap<Image::Type, Owned<Store>>> Store::create(
const Flags& flags,
SecretResolver* secretResolver)
{
+#ifdef __WINDOWS__
+ if (!flags.image_providers.isNone()) {
+ return Error("Image stores are not supported on Windows");
+ }
+
+ return hashmap<Image::Type, Owned<Store>>();
+#else
if (flags.image_providers.isNone()) {
return hashmap<Image::Type, Owned<Store>>();
}
@@ -86,6 +95,7 @@ Try<hashmap<Image::Type, Owned<Store>>> Store::create(
}
return stores;
+#endif // __WINDOWS__
}
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index 3ea8b44..5eb9c65 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -50,7 +50,6 @@ set(MESOS_TESTS_UTILS_SRC
main.cpp
master/mock_master_api_subscriber.cpp
mesos.cpp
- mock_csi_plugin.cpp
mock_docker.cpp
mock_fetcher.cpp
mock_registrar.cpp
@@ -63,6 +62,7 @@ set(MESOS_TESTS_UTILS_SRC
if (NOT WIN32)
list(APPEND MESOS_TESTS_UTILS_SRC
kill_policy_test_helper.cpp
+ mock_csi_plugin.cpp
script.cpp
containerizer/memory_test_helper.cpp)
endif ()
@@ -91,8 +91,6 @@ set(MESOS_TESTS_SRC
container_daemon_tests.cpp
cram_md5_authentication_tests.cpp
credentials_tests.cpp
- csi_client_tests.cpp
- csi_utils_tests.cpp
default_executor_tests.cpp
exception_tests.cpp
executor_http_api_tests.cpp
@@ -162,13 +160,14 @@ list(APPEND MESOS_TESTS_SRC
containerizer/containerizer_tests.cpp
containerizer/cpu_isolator_tests.cpp
containerizer/docker_containerizer_tests.cpp
- containerizer/docker_spec_tests.cpp
containerizer/docker_tests.cpp
containerizer/memory_isolator_tests.cpp)
if (NOT WIN32)
list(APPEND MESOS_TESTS_SRC
container_logger_tests.cpp
+ csi_client_tests.cpp
+ csi_utils_tests.cpp
disk_profile_adaptor_tests.cpp
disk_quota_tests.cpp
dynamic_weights_tests.cpp
@@ -204,6 +203,7 @@ if (NOT WIN32)
list(APPEND MESOS_TESTS_SRC
containerizer/appc_spec_tests.cpp
containerizer/composing_containerizer_tests.cpp
+ containerizer/docker_spec_tests.cpp
containerizer/environment_secret_isolator_tests.cpp
containerizer/io_switchboard_tests.cpp
containerizer/isolator_tests.cpp
diff --git a/src/tests/uri_fetcher_tests.cpp b/src/tests/uri_fetcher_tests.cpp
index c727cc5..92f8062 100644
--- a/src/tests/uri_fetcher_tests.cpp
+++ b/src/tests/uri_fetcher_tests.cpp
@@ -35,7 +35,9 @@
#include <stout/tests/utils.hpp>
+#ifndef __WINDOWS__
#include <mesos/docker/spec.hpp>
+#endif // __WINDOWS__
#include "uri/fetcher.hpp"
@@ -49,7 +51,9 @@ namespace http = process::http;
using std::list;
using std::string;
+#ifndef __WINDOWS__
using mesos::uri::DockerFetcherPlugin;
+#endif // __WINDOWS__
using process::Future;
using process::Owned;
@@ -306,6 +310,9 @@ static constexpr char TEST_DIGEST[] = "sha256:a3ed95caeb02ffe68cdd9fd844066"
#endif // __WINDOWS__
+// NOTE: Windows images exist and can be fetched, but there is no Windows
+// provisioner in the Mesos containerizer that can use these images.
+#ifndef __WINDOWS__
class DockerFetcherPluginTest : public TemporaryDirectoryTest {};
@@ -423,6 +430,7 @@ TEST_F(DockerFetcherPluginTest, DISABLED_INTERNET_CURL_InvokeFetchByName)
manifest->layers(i).digest())));
}
}
+#endif // __WINDOWS__
class CopyFetcherPluginTest : public TemporaryDirectoryTest {};
diff --git a/src/uri/fetcher.cpp b/src/uri/fetcher.cpp
index 8db43eb..54e30ce 100644
--- a/src/uri/fetcher.cpp
+++ b/src/uri/fetcher.cpp
@@ -52,8 +52,10 @@ Try<Owned<Fetcher>> create(const Option<Flags>& _flags)
[flags]() { return CopyFetcherPlugin::create(flags); }},
{HadoopFetcherPlugin::NAME,
[flags]() { return HadoopFetcherPlugin::create(flags); }},
+#ifndef __WINDOWS__
{DockerFetcherPlugin::NAME,
[flags]() { return DockerFetcherPlugin::create(flags); }},
+#endif // __WINDOWS__
};
vector<Owned<Fetcher::Plugin>> plugins;
diff --git a/src/uri/fetcher.hpp b/src/uri/fetcher.hpp
index cc4bd93..5b47b79 100644
--- a/src/uri/fetcher.hpp
+++ b/src/uri/fetcher.hpp
@@ -27,9 +27,12 @@
#include "uri/fetchers/copy.hpp"
#include "uri/fetchers/curl.hpp"
-#include "uri/fetchers/docker.hpp"
#include "uri/fetchers/hadoop.hpp"
+#ifndef __WINDOWS__
+#include "uri/fetchers/docker.hpp"
+#endif // __WINDOWS__
+
namespace mesos {
namespace uri {
namespace fetcher {
@@ -40,8 +43,10 @@ namespace fetcher {
class Flags :
public virtual CopyFetcherPlugin::Flags,
public virtual CurlFetcherPlugin::Flags,
- public virtual HadoopFetcherPlugin::Flags,
- public virtual DockerFetcherPlugin::Flags {};
+#ifndef __WINDOWS__
+ public virtual DockerFetcherPlugin::Flags,
+#endif // __WINDOWS__
+ public virtual HadoopFetcherPlugin::Flags {};
/**