You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by as...@apache.org on 2020/08/28 12:47:08 UTC
[mesos] branch master updated (62931a0 -> 355b971)
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 62931a0 Added basic tests for offer constraints on a string attribute equality.
new b2e12fc Added RE2 to the CMake build.
new afe493e Added RE2 to the automake build.
new a91747e Added protobuf messages for regex-based offer constraints.
new 470665e Implemented regex-based offer constraints.
new 355b971 Added basic tests for regexp-based constraints.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
3rdparty/CMakeLists.txt | 82 ++++++++
3rdparty/Makefile.am | 20 ++
3rdparty/cmake/Versions.cmake | 2 +
3rdparty/re2-2020-07-06.tar.gz | Bin 0 -> 404055 bytes
3rdparty/versions.am | 1 +
LICENSE | 32 +++
configure.ac | 50 +++++
include/mesos/allocator/allocator.hpp | 16 ++
include/mesos/scheduler/scheduler.proto | 53 ++++-
include/mesos/v1/scheduler/scheduler.proto | 53 ++++-
src/CMakeLists.txt | 1 +
src/Makefile.am | 12 ++
.../allocator/mesos/offer_constraints_filter.cpp | 107 +++++++++-
src/master/constants.hpp | 10 +
src/master/flags.cpp | 18 ++
src/master/flags.hpp | 3 +
src/master/master.cpp | 8 +-
src/master/master.hpp | 3 +
src/python/native_common/ext_modules.py.in | 9 +
.../master/offer_constraints_filter_tests.cpp | 219 +++++++++++++++++++--
20 files changed, 670 insertions(+), 29 deletions(-)
create mode 100644 3rdparty/re2-2020-07-06.tar.gz
[mesos] 01/05: Added RE2 to the CMake build.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b2e12fc423a5c1fcb786e2fce01193ef7207006e
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Wed Jul 15 20:01:25 2020 +0200
Added RE2 to the CMake build.
This is a prerequisite for implementing regex-based offer constraints.
Review: https://reviews.apache.org/r/72782
---
3rdparty/CMakeLists.txt | 82 +++++++++++++++++++++++++++++++++++++++++
3rdparty/cmake/Versions.cmake | 2 +
3rdparty/re2-2020-07-06.tar.gz | Bin 0 -> 404055 bytes
LICENSE | 32 ++++++++++++++++
src/CMakeLists.txt | 1 +
5 files changed, 117 insertions(+)
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
index af9a071..59c931c 100644
--- a/3rdparty/CMakeLists.txt
+++ b/3rdparty/CMakeLists.txt
@@ -42,6 +42,7 @@ set(NVML_URL ${FETCH_URL}/nvml-${NVML_VERSION}.tar.gz)
set(PICOJSON_URL ${FETCH_URL}/picojson-${PICOJSON_VERSION}.tar.gz)
set(PROTOBUF_URL ${FETCH_URL}/protobuf-${PROTOBUF_VERSION}.tar.gz)
set(RAPIDJSON_URL ${FETCH_URL}/rapidjson-${RAPIDJSON_VERSION}.tar.gz)
+set(RE2_URL ${FETCH_URL}/re2-${RE2_VERSION}.tar.gz)
set(XZ_URL ${FETCH_URL}/xz-${XZ_VERSION}-modified.tar.gz)
set(ZOOKEEPER_URL ${FETCH_URL}/zookeeper-${ZOOKEEPER_VERSION}.tar.gz)
@@ -477,6 +478,87 @@ if (ENABLE_INSTALL_MODULE_DEPENDENCIES)
DESTINATION ${MESOS_INSTALL_HEADERS})
endif ()
+# re2: non-backtracking regular expression library by Google
+# https://github.com/google/re2
+################################
+EXTERNAL(re2 ${RE2_VERSION} ${CMAKE_CURRENT_BINARY_DIR})
+add_library(re2 ${LIBRARY_LINKAGE} IMPORTED GLOBAL)
+add_dependencies(re2 ${RE2_TARGET})
+
+# NOTE: Mesos build uses re2 installed into a prefix (similarly to glog).
+set(RE2_INSTALL_DIR ${RE2_ROOT}-install)
+
+set(
+ RE2_CMAKE_ARGS
+ ${CMAKE_CXX_FORWARD_ARGS}
+ -DBUILD_TESTING=OFF
+ -DCMAKE_INSTALL_BINDIR=${RE2_INSTALL_DIR}/bin
+ -DCMAKE_INSTALL_INCLUDEDIR=${RE2_INSTALL_DIR}/include
+ -DCMAKE_INSTALL_LIBDIR=${RE2_INSTALL_DIR}/lib
+)
+
+if (WIN32)
+ if (BUILD_SHARED_LIBS)
+ set(RE2_IMPORTED_LOCATION_DIR ${RE2_INSTALL_DIR}/bin)
+ else()
+ set(RE2_IMPORTED_LOCATION_DIR ${RE2_INSTALL_DIR}/lib)
+ endif()
+
+ if (CMAKE_GENERATOR MATCHES "Visual Studio")
+ # TODO(asekretenko): Make sure that these locations are correct on
+ # MSVC Windows both in debug and release modes; adjust if necessary.
+ set_target_properties(
+ re2 PROPERTIES
+ IMPORTED_LOCATION_DEBUG ${RE2_IMPORTED_LOCATION_DIR}/re2${LIBRARY_SUFFIX}
+ IMPORTED_LOCATION_RELEASE ${RE2_IMPORTED_LOCATION_DIR}/re2${LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_DEBUG ${RE2_INSTALL_DIR}/lib/re2${CMAKE_IMPORT_LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB_RELEASE ${RE2_INSTALL_DIR}/lib/re2${CMAKE_IMPORT_LIBRARY_SUFFIX}
+ )
+ else ()
+ # TODO(asekretenko): Make sure that these locations are correct on
+ # non-MSVC Windows; adjust if necessary.
+ set_target_properties(
+ re2 PROPERTIES
+ IMPORTED_LOCATION ${RE2_IMPORTED_LOCATION_DIR}/libre2${LIBRARY_SUFFIX}
+ IMPORTED_IMPLIB ${RE2_INSTALL_DIR}/lib/libre2${CMAKE_IMPORT_LIBRARY_SUFFIX}
+ )
+ endif()
+else()
+ set_target_properties(
+ re2 PROPERTIES
+ IMPORTED_LOCATION ${RE2_INSTALL_DIR}/lib/libre2${LIBRARY_SUFFIX}
+ )
+endif ()
+
+set_target_properties(
+ re2 PROPERTIES
+ INTERFACE_INCLUDE_DIRECTORIES ${RE2_INSTALL_DIR}/include
+)
+
+MAKE_INCLUDE_DIR(re2)
+GET_BYPRODUCTS(re2)
+
+ExternalProject_Add(
+ ${RE2_TARGET}
+ PREFIX ${RE2_CMAKE_ROOT}
+ BUILD_BYPRODUCTS ${RE2_BYPRODUCTS}
+ CMAKE_ARGS ${RE2_CMAKE_ARGS}
+ INSTALL_DIR ${RE2_INSTALL_DIR}
+ URL ${RE2_URL}
+ URL_HASH ${RE2_HASH})
+
+if (WIN32 AND BUILD_SHARED_LIBS)
+ # TODO(asekretenko): Make sure that this location is correct on
+ # Windows; adjust if necessary.
+ install(
+ DIRECTORY ${RE2_INSTALL_DIR}/bin/
+ DESTINATION ${MESOS_INSTALL_RUNTIME})
+endif()
+
+install(
+ DIRECTORY ${RE2_INSTALL_DIR}/lib/
+ DESTINATION ${MESOS_INSTALL_LIBRARIES})
+
# PicoJSON: JSON parser / serializer.
# https://github.com/kazuho/picojson
#####################################
diff --git a/3rdparty/cmake/Versions.cmake b/3rdparty/cmake/Versions.cmake
index 9376616..eafe255 100644
--- a/3rdparty/cmake/Versions.cmake
+++ b/3rdparty/cmake/Versions.cmake
@@ -49,6 +49,8 @@ set(ZOOKEEPER_VERSION "3.4.8")
set(ZOOKEEPER_HASH "SHA256=F10A0B51F45C4F64C1FE69EF713ABF9EB9571BC7385A82DA892E83BB6C965E90")
set(GLOG_VERSION "0.4.0")
set(GLOG_HASH "SHA256=F28359AEBA12F30D73D9E4711EF356DC842886968112162BC73002645139C39C")
+set(RE2_VERSION "2020-07-06")
+set(RE2_HASH "SHA256=2E9489A31AE007C81E90E8EC8A15D62D58A9C18D4FD1603F6441EF248556B41F")
# Platform-dependent versions.
if (WIN32)
diff --git a/3rdparty/re2-2020-07-06.tar.gz b/3rdparty/re2-2020-07-06.tar.gz
new file mode 100644
index 0000000..a7d6c9f
Binary files /dev/null and b/3rdparty/re2-2020-07-06.tar.gz differ
diff --git a/LICENSE b/LICENSE
index 2c5919d..6ef484e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1295,3 +1295,35 @@ RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES.
+
+
+======================================================================
+For re2-2020-07-06 (3rdparty/re2-2020-07-06.tar.gz):
+======================================================================
+// Copyright (c) 2009 The RE2 Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 668a506..f5105c7 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -634,6 +634,7 @@ target_link_libraries(
if (NOT WIN32)
target_link_libraries(mesos PUBLIC leveldb)
+ target_link_libraries(mesos PUBLIC re2)
endif ()
if (ENABLE_SECCOMP_ISOLATOR)
[mesos] 03/05: Added protobuf messages for regex-based offer
constraints.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit a91747eaf60351507f16d8d167a0fa0207a9811a
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Thu Jul 16 15:48:03 2020 +0200
Added protobuf messages for regex-based offer constraints.
This patch adds protobuf messages for setting offer constraints
that check if agent's (pseudo)attribute matches a specified RE2 regular
expression.
Both added contsraint predicates will evaluate to `true` when the
attribute is not TEXT. This way, schedulers will have to apply on their
own whatever filtration they do for non-TEXT attributes which happen
to be selected by the constraint's `Selector`.
Given that in the real world schedulers seem to rarely put constraints
on attributes that are normally Scalar/Ranges, this should not prevent
them from obtaining performance benefits by setting offer constraints.
Review: https://reviews.apache.org/r/72784
---
include/mesos/scheduler/scheduler.proto | 53 ++++++++++++++++++++++++++++--
include/mesos/v1/scheduler/scheduler.proto | 53 ++++++++++++++++++++++++++++--
2 files changed, 102 insertions(+), 4 deletions(-)
diff --git a/include/mesos/scheduler/scheduler.proto b/include/mesos/scheduler/scheduler.proto
index 06d62f5..f70738c 100644
--- a/include/mesos/scheduler/scheduler.proto
+++ b/include/mesos/scheduler/scheduler.proto
@@ -325,8 +325,54 @@ message AttributeConstraint {
required string value = 1;
}
- // TODO(asekretenko): add predicates for regular expression matching
- // (MESOS-10173).
+ // Predicates for regular expression constraints.
+ //
+ // The regular expressions are interpreted according to
+ // RE2 regular expression syntax and semantics:
+ // https://github.com/google/re2/blob/master/doc/syntax.txt
+ //
+ // The call using the constraints is invalid if the specified string is not
+ // a valid RE2 regex, or the regex is deemed too expensive to
+ // construct/store/execute, namely:
+ //
+ // - the RE2 object cannot be constructed from the regex without breaching
+ // the limit on the estimated memory footprint of an individual RE2
+ // (controlled by the --offer_constraints_re2_max_mem master flag), or
+ //
+ // - the regex program size reported by `RE2::ProgramSize()`
+ // exceeds the value of --offer_constraints_re2_max_program_size flag.
+ //
+ // Frameworks using the C++ implementation of RE2 (directly or via the JNI
+ // wrapper) for regexp validation on their side, are advised to set
+ // the `max_mem` RE2 option and limit `ProgramSize()` to values no larger
+ // than the defaults for the corresonding Mesos master flags.
+ //
+ // TODO(asekretenko): Provide an API for the frameworks and/or operators
+ // to validate regexp constraints against the Mesos master setup.
+ //
+ // Similarly to the string equality predicates, both regexp predicates match
+ // (evaluate to `true`) when the (pseudo)attribute is not a TEXT/string.
+ // This way, schedulers will have to apply purely on their own
+ // whatever filtration they do for non-TEXT attributes which happen
+ // to be selected by the constraint's `Selector`
+
+ // Yields `true` if the (pseudo)attribute exists and is a TEXT/string
+ // matching the specified regex.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reasons explained above.
+ message TextMatches {
+ required string regex = 1;
+ }
+
+ // Yields `true` if the (pseudo)attribute either does not exists or is
+ // not a TEXT/string matching the specified regex.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reasons explained above.
+ message TextNotMatches {
+ required string regex = 1;
+ }
oneof predicate {
Exists exists = 1;
@@ -334,6 +380,9 @@ message AttributeConstraint {
TextEquals text_equals = 3;
TextNotEquals text_not_equals = 4;
+
+ TextMatches text_matches = 5;
+ TextNotMatches text_not_matches = 6;
}
}
diff --git a/include/mesos/v1/scheduler/scheduler.proto b/include/mesos/v1/scheduler/scheduler.proto
index 08fb10a..364d2c8 100644
--- a/include/mesos/v1/scheduler/scheduler.proto
+++ b/include/mesos/v1/scheduler/scheduler.proto
@@ -323,8 +323,54 @@ message AttributeConstraint {
required string value = 1;
}
- // TODO(asekretenko): add predicates for regular expression matching
- // (MESOS-10173).
+ // Predicates for regular expression constraints.
+ //
+ // The regular expressions are interpreted according to
+ // RE2 regular expression syntax and semantics:
+ // https://github.com/google/re2/blob/master/doc/syntax.txt
+ //
+ // The call using the constraints is invalid if the specified string is not
+ // a valid RE2 regex, or the regex is deemed too expensive to
+ // construct/store/execute, namely:
+ //
+ // - the RE2 object cannot be constructed from the regex without breaching
+ // the limit on the estimated memory footprint of an individual RE2
+ // (controlled by the --offer_constraints_re2_max_mem master flag), or
+ //
+ // - the regex program size reported by `RE2::ProgramSize()`
+ // exceeds the value of --offer_constraints_re2_max_program_size flag.
+ //
+ // Frameworks using the C++ implementation of RE2 (directly or via the JNI
+ // wrapper) for regexp validation on their side, are advised to set
+ // the `max_mem` RE2 option and limit `ProgramSize()` to values no larger
+ // than the defaults for the corresonding Mesos master flags.
+ //
+ // TODO(asekretenko): Provide an API for the frameworks and/or operators
+ // to validate regexp constraints against the Mesos master setup.
+ //
+ // Similarly to the string equality predicates, both regexp predicates match
+ // (evaluate to `true`) when the (pseudo)attribute is not a TEXT/string.
+ // This way, schedulers will have to apply purely on their own
+ // whatever filtration they do for non-TEXT attributes which happen
+ // to be selected by the constraint's `Selector`
+
+ // Yields `true` if the (pseudo)attribute exists and is a TEXT/string
+ // matching the specified regex.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reasons explained above.
+ message TextMatches {
+ required string regex = 1;
+ }
+
+ // Yields `true` if the (pseudo)attribute either does not exists or is
+ // not a TEXT/string matching the specified regex.
+ //
+ // Always yields `true` for existing non-TEXT attributes (and non-string
+ // pseudoattributes) for the reasons explained above.
+ message TextNotMatches {
+ required string regex = 1;
+ }
oneof predicate {
Exists exists = 1;
@@ -332,6 +378,9 @@ message AttributeConstraint {
TextEquals text_equals = 3;
TextNotEquals text_not_equals = 4;
+
+ TextMatches text_matches = 5;
+ TextNotMatches text_not_matches = 6;
}
}
[mesos] 02/05: Added RE2 to the automake build.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit afe493e73a2b954d1b2dba3d4c5f6b08b9cb0b1b
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Mon Aug 17 15:54:43 2020 +0200
Added RE2 to the automake build.
This is a prerequisite to implementing regex-based offer constaints.
Review: https://reviews.apache.org/r/72783
---
3rdparty/Makefile.am | 20 ++++++++++++
3rdparty/versions.am | 1 +
configure.ac | 50 ++++++++++++++++++++++++++++++
src/Makefile.am | 11 +++++++
src/python/native_common/ext_modules.py.in | 9 ++++++
5 files changed, 91 insertions(+)
diff --git a/3rdparty/Makefile.am b/3rdparty/Makefile.am
index c277627..23f49ec 100644
--- a/3rdparty/Makefile.am
+++ b/3rdparty/Makefile.am
@@ -71,6 +71,7 @@ PIP = pip-$(PIP_VERSION)
PICOJSON = picojson-$(PICOJSON_VERSION)
PROTOBUF = protobuf-$(PROTOBUF_VERSION)
RAPIDJSON = rapidjson-$(RAPIDJSON_VERSION)
+RE2 = re2-$(RE2_VERSION)
SETUPTOOLS = setuptools-$(SETUPTOOLS_VERSION)
WHEEL = wheel-$(WHEEL_VERSION)
ZOOKEEPER = zookeeper-$(ZOOKEEPER_VERSION)
@@ -97,6 +98,7 @@ EXTRA_DIST = \
$(PROTOBUF).tar.gz \
$(PICOJSON).tar.gz \
$(RAPIDJSON).tar.gz \
+ $(RE2).tar.gz \
$(SETUPTOOLS).tar.gz \
$(WHEEL).tar.gz \
$(ZOOKEEPER).tar.gz
@@ -167,6 +169,7 @@ CLEAN_EXTRACTED = \
$(PICOJSON) \
$(PROTOBUF) \
$(RAPIDJSON) \
+ $(RE2) \
$(SETUPTOOLS) \
$(WHEEL) \
$(ZOOKEEPER)
@@ -619,6 +622,19 @@ $(LEVELDB)/out-static/libleveldb.a: $(LEVELDB)-stamp
ALL_LOCAL += $(LEVELDB)/out-static/libleveldb.a
endif
+if WITH_BUNDLED_RE2
+LIB_RE2 = $(RE2)/obj/libre2.a
+
+$(LIB_RE2): $(RE2)-build-stamp
+
+$(RE2)-build-stamp: $(RE2)-stamp
+ cd $(RE2) && $(MAKE) $(AM_MAKEFLAGS) CXX="$(CXX)" CXXFLAGS="$(CXXFLAGS) -fPIC"
+ touch $@
+
+ALL_LOCAL += $(LIB_RE2)
+endif
+
+
if WITH_BUNDLED_ZOOKEEPER
$(ZOOKEEPER)/src/c/libzookeeper_mt.la: $(ZOOKEEPER)-stamp
cd $(ZOOKEEPER)/src/c && ./configure $(CONFIGURE_ARGS) && \
@@ -697,6 +713,10 @@ if WITH_BUNDLED_RAPIDJSON
rm -rf $(INSTALLDIR)/include/rapidjson
cp -fpR $(RAPIDJSON)/include/rapidjson $(INSTALLDIR)/include/
endif
+if WITH_BUNDLED_RE2
+ cd $(RE2) && \
+ $(MAKE) $(AM_MAKEFLAGS) DESTDIR=$(INSTALLDIR) install
+endif
if WITH_BUNDLED_ZOOKEEPER
cd $(ZOOKEEPER)/src/c && \
$(MAKE) $(AM_MAKEFLAGS) DESTDIR=$(INSTALLDIR) install
diff --git a/3rdparty/versions.am b/3rdparty/versions.am
index 6f6195d..5b90be8 100644
--- a/3rdparty/versions.am
+++ b/3rdparty/versions.am
@@ -40,6 +40,7 @@ PICOJSON_VERSION = 1.3.0
PIP_VERSION = 7.1.2
PROTOBUF_VERSION = 3.5.0
RAPIDJSON_VERSION = 1.1.0
+RE2_VERSION = 2020-07-06
SETUPTOOLS_VERSION = 20.9.0
WHEEL_VERSION = 0.24.0
ZOOKEEPER_VERSION = 3.4.8
diff --git a/configure.ac b/configure.ac
index a049945..b19440a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -528,6 +528,13 @@ AC_ARG_WITH([rapidjson],
location prefixed by the given path]),
[without_bundled_rapidjson=yes], [])
+AC_ARG_WITH([re2],
+ AS_HELP_STRING([--with-re2@<:@=DIR@:>@],
+ [excludes building and using the bundled re2
+ package in lieu of an installed version at a
+ location prefixed by the given path]),
+ [without_bundled_re2=yes], [])
+
AC_ARG_WITH([protobuf],
AS_HELP_STRING([--with-protobuf@<:@=DIR@:>@],
[excludes building and using the bundled protobuf
@@ -2019,6 +2026,49 @@ fi
AM_CONDITIONAL([WITH_BUNDLED_RAPIDJSON], [test "x$with_bundled_rapidjson" = "xyes"])
+# Check if re2 prefix path was supplied and if so, add it to CPPFLAGS
+# while extending it by /include and to LDFLAGS while extending it by
+# /lib.
+if test -n "`echo $with_re2`"; then
+ CPPFLAGS="$CPPFLAGS -I${with_re2}/include"
+ LDFLAGS="$LDFLAGS -L${with_re2}/lib"
+fi
+
+# Check if user has asked us to use a preinstalled re2, or if they
+# asked us to ignore all bundled libraries while compiling and
+# linking.
+if test "x$without_bundled_re2" = "xyes" || \
+ test "x$enable_bundled" != "xyes"; then
+ # Check if headers and library were located.
+ AC_CHECK_HEADERS([re2/re2.h],
+ [AC_LINK_IFELSE(
+ [AC_LANG_PROGRAM([#include <re2/re2.h>], [])],
+ [TEST_LIBS="$TEST_LIBS -lre2"] [found_re2=yes],
+ [AC_MSG_WARN([re2 is not installed.])])])
+
+ if test "x$found_re2" = "xyes"; then
+ with_bundled_re2=no
+ else
+ AC_MSG_ERROR([cannot find re2
+-------------------------------------------------------------------
+You have requested the use of a non-bundled re2 but no suitable
+re2 could be found.
+
+You may want specify the location of re2 by providing a prefix
+path via --with-re2=DIR, or check that the path you provided is
+correct if you're already doing this.
+-------------------------------------------------------------------
+])
+ fi
+else
+ with_bundled_re2=yes
+fi
+
+AM_CONDITIONAL([WITH_BUNDLED_RE2],
+ [test "x$with_bundled_re2" = "xyes"])
+
+
+
# Check if Sasl2 prefix path was provided, and if so, add it to
# the CPPFLAGS and LDFLAGS with respective /include and /lib path
diff --git a/src/Makefile.am b/src/Makefile.am
index 8b95611..1043c7b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -44,6 +44,7 @@ PICOJSON = 3rdparty/picojson-$(PICOJSON_VERSION)
PIP = 3rdparty/pip-$(PIP_VERSION)
PROTOBUF = 3rdparty/protobuf-$(PROTOBUF_VERSION)
RAPIDJSON = 3rdparty/rapidjson-$(RAPIDJSON_VERSION)
+RE2 = 3rdparty/re2-$(RE2_VERSION)
SETUPTOOLS = 3rdparty/setuptools-$(SETUPTOOLS_VERSION)
STOUT = 3rdparty/stout
WHEEL = 3rdparty/wheel-$(WHEEL_VERSION)
@@ -265,6 +266,15 @@ if WITH_BUNDLED_RAPIDJSON
MESOS_CPPFLAGS += -I../$(RAPIDJSON)/include
endif
+if WITH_BUNDLED_RE2
+MESOS_CPPFLAGS += -I../$(RE2)
+LIB_RE2 = ../$(RE2)/obj/libre2.a
+else
+LIB_RE2 = -lre2
+LDADD += -lre2
+endif
+
+
if WITH_BUNDLED_STOUT
MESOS_CPPFLAGS += -I$(top_srcdir)/$(STOUT)/include
endif
@@ -1745,6 +1755,7 @@ libmesos_la_LIBADD = \
$(LIB_PROCESS) \
$(LIB_PROTOBUF) \
$(LIB_LIBSECCOMP) \
+ $(LIB_RE2) \
$(LIB_ZOOKEEPER) \
-lsvn_subr-1 \
-lsvn_delta-1 \
diff --git a/src/python/native_common/ext_modules.py.in b/src/python/native_common/ext_modules.py.in
index 38e6717..62e32a6 100644
--- a/src/python/native_common/ext_modules.py.in
+++ b/src/python/native_common/ext_modules.py.in
@@ -115,6 +115,8 @@ def _create_module(module_name):
gperftools = os.path.join('3rdparty', 'gperftools-2.5')
protobuf = os.path.join('3rdparty', 'protobuf-3.5.0')
+ re2 = os.path.join('3rdparty', 're2-2020-07-06')
+
# Build the list of source files. Note that each source must be
# relative to our current directory (where this script lives).
SOURCES = [
@@ -169,6 +171,13 @@ def _create_module(module_name):
'-lprotobuf'
)
+ EXTRA_OBJECTS += _cond_extra_object(
+ "@WITH_BUNDLED_RE2_TRUE@",
+ "@WITH_BUNDLED_RE2_FALSE@",
+ os.path.join(abs_top_builddir, re2, 'obj', 'libre2.a'),
+ '-lre2'
+ )
+
if '@ENABLE_SECCOMP_ISOLATOR_TRUE@' == '':
libseccomp = os.path.join('3rdparty', 'libseccomp-2.3.3')
libseccomp = os.path.join(
[mesos] 04/05: Implemented regex-based offer constraints.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 470665e1575b9a91dffca974485940c7a0a1fe37
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Mon Aug 3 18:35:06 2020 +0200
Implemented regex-based offer constraints.
Review: https://reviews.apache.org/r/72786
---
include/mesos/allocator/allocator.hpp | 16 +++
src/Makefile.am | 1 +
.../allocator/mesos/offer_constraints_filter.cpp | 107 +++++++++++++++++++--
src/master/constants.hpp | 10 ++
src/master/flags.cpp | 18 ++++
src/master/flags.hpp | 3 +
src/master/master.cpp | 8 +-
src/master/master.hpp | 3 +
.../master/offer_constraints_filter_tests.cpp | 33 ++++---
9 files changed, 174 insertions(+), 25 deletions(-)
diff --git a/include/mesos/allocator/allocator.hpp b/include/mesos/allocator/allocator.hpp
index 05d0e9c..c6fca65 100644
--- a/include/mesos/allocator/allocator.hpp
+++ b/include/mesos/allocator/allocator.hpp
@@ -78,7 +78,23 @@ class OfferConstraintsFilterImpl;
class OfferConstraintsFilter
{
public:
+ // TODO(asekretenko): Given that most dependants of the public allocator
+ // interface do not care about filter creation, we should consider decoupling
+ // the filter construction interface (which at this point consists of the
+ // `struct Options` and the `create()` method) from the allocator interface.
+ struct Options
+ {
+ struct RE2Limits
+ {
+ Bytes maxMem;
+ int maxProgramSize;
+ };
+
+ RE2Limits re2Limits;
+ };
+
static Try<OfferConstraintsFilter> create(
+ const Options& options,
scheduler::OfferConstraints&& constraints);
OfferConstraintsFilter() = delete;
diff --git a/src/Makefile.am b/src/Makefile.am
index 1043c7b..943ed77 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1133,6 +1133,7 @@ libmesos_no_3rdparty_la_SOURCES += \
master/allocator/mesos/hierarchical.hpp \
master/allocator/mesos/metrics.cpp \
master/allocator/mesos/metrics.hpp \
+ master/allocator/mesos/offer_constraints_filter.hpp \
master/allocator/mesos/offer_constraints_filter.cpp \
master/allocator/mesos/sorter/drf/metrics.cpp \
master/allocator/mesos/sorter/drf/metrics.hpp \
diff --git a/src/master/allocator/mesos/offer_constraints_filter.cpp b/src/master/allocator/mesos/offer_constraints_filter.cpp
index d724fd0..30b671b 100644
--- a/src/master/allocator/mesos/offer_constraints_filter.cpp
+++ b/src/master/allocator/mesos/offer_constraints_filter.cpp
@@ -23,6 +23,8 @@
#include <stout/try.hpp>
#include <stout/variant.hpp>
+#include <re2/re2.h>
+
#include <mesos/allocator/allocator.hpp>
#include <mesos/attributes.hpp>
@@ -34,11 +36,42 @@ using std::unordered_map;
using ::mesos::scheduler::AttributeConstraint;
using ::mesos::scheduler::OfferConstraints;
+using RE2Limits =
+ ::mesos::allocator::OfferConstraintsFilter::Options::RE2Limits;
+
namespace mesos {
namespace allocator {
namespace internal {
+// TODO(asekretenko): Cache returned RE2s to make sure that:
+// - a RE2 corresponding to a pattern is not re-created needlessly
+// - multiple RE2s corresponding to the same pattern don't coexist
+static Try<unique_ptr<const RE2>> createRE2(
+ const RE2Limits& limits,
+ const string& regex)
+{
+ RE2::Options options{RE2::CannedOptions::Quiet};
+ options.set_max_mem(limits.maxMem.bytes());
+ unique_ptr<const RE2> re2{new RE2(regex, options)};
+
+ if (!re2->ok()) {
+ return Error(
+ "Failed to construct regex from pattern"
+ " '" + regex + "': " + re2->error());
+ }
+
+ if (re2->ProgramSize() > limits.maxProgramSize) {
+ return Error(
+ "Regex '" + regex + "' is too complex: program size of " +
+ stringify(re2->ProgramSize()) + " is larger than maximum of " +
+ stringify(limits.maxProgramSize) + " allowed");
+ }
+
+ return re2;
+}
+
+
using Selector = AttributeConstraint::Selector;
@@ -78,6 +111,7 @@ public:
bool apply(const Attribute& attr) const { return apply_(attr); }
static Try<AttributeConstraintPredicate> create(
+ const RE2Limits& re2Limits,
AttributeConstraint::Predicate&& predicate)
{
using Self = AttributeConstraintPredicate;
@@ -96,6 +130,28 @@ public:
return Self(TextNotEquals{
std::move(*predicate.mutable_text_not_equals()->mutable_value())});
+ case AttributeConstraint::Predicate::kTextMatches: {
+ Try<unique_ptr<const RE2>> re2 =
+ createRE2(re2Limits, predicate.text_matches().regex());
+
+ if (re2.isError()) {
+ return Error(re2.error());
+ }
+
+ return Self(TextMatches{std::move(*re2)});
+ }
+
+ case AttributeConstraint::Predicate::kTextNotMatches: {
+ Try<unique_ptr<const RE2>> re2 =
+ createRE2(re2Limits, predicate.text_not_matches().regex());
+
+ if (re2.isError()) {
+ return Error(re2.error());
+ }
+
+ return Self(TextNotMatches{std::move(*re2)});
+ }
+
case AttributeConstraint::Predicate::PREDICATE_NOT_SET:
return Error("Unknown predicate type");
}
@@ -146,15 +202,43 @@ private:
}
};
- // TODO(asekretenko): Introduce offer constraints for regex match
- // (MESOS-10173).
+ struct TextMatches
+ {
+ unique_ptr<const RE2> re2;
+
+ bool apply(const Nothing&) const { return false; }
+ bool apply(const string& str) const { return RE2::FullMatch(str, *re2); }
+
+ bool apply(const Attribute& attr) const
+ {
+ return attr.type() != Value::TEXT ||
+ RE2::FullMatch(attr.text().value(), *re2);
+ }
+ };
+
+ struct TextNotMatches
+ {
+ unique_ptr<const RE2> re2;
+
+ bool apply(const Nothing&) const { return true; }
+ bool apply(const string& str) const { return !RE2::FullMatch(str, *re2); }
+
+ bool apply(const Attribute& attr) const
+ {
+ return attr.type() != Value::TEXT ||
+ !RE2::FullMatch(attr.text().value(), *re2);
+ }
+ };
+
using Predicate = Variant<
Nothing,
Exists,
NotExists,
TextEquals,
- TextNotEquals>;
+ TextNotEquals,
+ TextMatches,
+ TextNotMatches>;
Predicate predicate;
@@ -171,7 +255,9 @@ private:
[&](const Exists& p) { return p.apply(attribute); },
[&](const NotExists& p) { return p.apply(attribute); },
[&](const TextEquals& p) { return p.apply(attribute); },
- [&](const TextNotEquals& p) { return p.apply(attribute); });
+ [&](const TextNotEquals& p) { return p.apply(attribute); },
+ [&](const TextMatches& p) { return p.apply(attribute); },
+ [&](const TextNotMatches& p) { return p.apply(attribute); });
}
};
@@ -223,6 +309,7 @@ public:
}
static Try<AttributeConstraintEvaluator> create(
+ const RE2Limits& re2Limits,
AttributeConstraint&& constraint)
{
Option<Error> error = validate(constraint.selector());
@@ -232,7 +319,7 @@ public:
Try<AttributeConstraintPredicate> predicate =
AttributeConstraintPredicate::create(
- std::move(*constraint.mutable_predicate()));
+ re2Limits, std::move(*constraint.mutable_predicate()));
if (predicate.isError()) {
return Error(predicate.error());
@@ -293,7 +380,9 @@ public:
});
}
- static Try<OfferConstraintsFilterImpl> create(OfferConstraints&& constraints)
+ static Try<OfferConstraintsFilterImpl> create(
+ const OfferConstraintsFilter::Options& options,
+ OfferConstraints&& constraints)
{
// TODO(asekretenko): This method performs a dumb 1:1 translation of
// `AttributeConstraint`s without any reordering; this leaves room for
@@ -333,7 +422,8 @@ public:
for (AttributeConstraint& constraint :
*group_.mutable_attribute_constraints()) {
Try<AttributeConstraintEvaluator> evaluator =
- AttributeConstraintEvaluator::create(std::move(constraint));
+ AttributeConstraintEvaluator::create(
+ options.re2Limits, std::move(constraint));
if (evaluator.isError()) {
return Error(
@@ -360,10 +450,11 @@ using internal::OfferConstraintsFilterImpl;
Try<OfferConstraintsFilter> OfferConstraintsFilter::create(
+ const Options& options,
OfferConstraints&& constraints)
{
Try<OfferConstraintsFilterImpl> impl =
- OfferConstraintsFilterImpl::create(std::move(constraints));
+ OfferConstraintsFilterImpl::create(options, std::move(constraints));
if (impl.isError()) {
return Error(impl.error());
diff --git a/src/master/constants.hpp b/src/master/constants.hpp
index c384b68..9aa075d 100644
--- a/src/master/constants.hpp
+++ b/src/master/constants.hpp
@@ -179,6 +179,16 @@ const Quota DEFAULT_QUOTA;
// Default weight for a role.
constexpr double DEFAULT_WEIGHT = 1.0;
+// Default values for the `max_mem` option and the limit on `RE2::ProgramSize()`
+// of RE2 regualr expressions used in offer constraints.
+//
+// As an example, for a regexp
+// "192.168.(1[0-9]|3[4-7]|[1-9]|4[2-9]|[1-4][0-9]|5[3-8]|20[4-7]|53[0-5]).1"
+// re2-2020-07-06 produces a RE2 object with a ProgramSize() of 54,
+// and that can be successfully constructed only with `max_mem` >= 1499.
+constexpr Bytes DEFAULT_OFFER_CONSTRAINTS_RE2_MAX_MEM = Bytes(4096);
+constexpr int DEFAULT_OFFER_CONSTRAINTS_RE2_MAX_PROGRAM_SIZE = 100;
+
} // namespace master {
} // namespace internal {
} // namespace mesos {
diff --git a/src/master/flags.cpp b/src/master/flags.cpp
index 74f4daa..31a8da1 100644
--- a/src/master/flags.cpp
+++ b/src/master/flags.cpp
@@ -743,4 +743,22 @@ mesos::internal::master::Flags::Flags()
}
return None();
});
+
+ add(&Flags::offer_constraints_re2_max_mem,
+ "offer_constraints_re2_max_mem",
+ "Limit on the memory usage of each RE2 regular expression in\n"
+ "framework's offer constraints. If `OfferConstraints` contain a regex\n"
+ "from which a RE2 object cannot be constructed without exceeding this\n"
+ "limit, then framework's attempt to subscribe or update subscription\n"
+ "with these `OfferConstraints` will fail.",
+ DEFAULT_OFFER_CONSTRAINTS_RE2_MAX_MEM);
+
+ add(&Flags::offer_constraints_re2_max_program_size,
+ "offer_constraints_re2_max_program_size",
+ "Limit on the RE2 program size of each regular expression in\n"
+ "framework's offer constraints. If `OfferConstraints` contain a regex\n"
+ "which results in a RE2 object exceeding this limit,\n"
+ "then framework's attempt to subscribe or update subscription\n"
+ "with these `OfferConstraints` will fail.",
+ DEFAULT_OFFER_CONSTRAINTS_RE2_MAX_PROGRAM_SIZE);
}
diff --git a/src/master/flags.hpp b/src/master/flags.hpp
index 78623d6..9500a0a 100644
--- a/src/master/flags.hpp
+++ b/src/master/flags.hpp
@@ -121,6 +121,9 @@ public:
#ifdef ENABLE_PORT_MAPPING_ISOLATOR
Option<size_t> max_executors_per_agent;
#endif // ENABLE_PORT_MAPPING_ISOLATOR
+
+ Bytes offer_constraints_re2_max_mem;
+ int offer_constraints_re2_max_program_size;
};
} // namespace master {
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 4428985..e3858b9 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -333,7 +333,10 @@ Master::Master(
subscribers(this, flags.max_operator_event_stream_subscribers),
authenticator(None()),
metrics(new Metrics(*this)),
- electedTime(None())
+ electedTime(None()),
+ offerConstraintsFilterOptions(
+ {flags.offer_constraints_re2_max_mem,
+ flags.offer_constraints_re2_max_program_size})
{
slaves.limiter = _slaveRemovalLimiter;
@@ -2671,6 +2674,7 @@ void Master::subscribe(
// TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
if (validationError.isNone() && subscribe.has_offer_constraints()) {
Try<OfferConstraintsFilter> filter = OfferConstraintsFilter::create(
+ offerConstraintsFilterOptions,
std::move(*subscribe.mutable_offer_constraints()));
if (filter.isError()) {
@@ -2913,6 +2917,7 @@ void Master::subscribe(
// TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
if (validationError.isNone() && subscribe.has_offer_constraints()) {
Try<OfferConstraintsFilter> filter = OfferConstraintsFilter::create(
+ offerConstraintsFilterOptions,
std::move(*subscribe.mutable_offer_constraints()));
if (filter.isError()) {
@@ -3253,6 +3258,7 @@ Future<process::http::Response> Master::updateFramework(
if (call.has_offer_constraints()) {
// TODO(asekretenko): Validate roles in offer constraints (see MESOS-10176).
Try<OfferConstraintsFilter> filter = OfferConstraintsFilter::create(
+ offerConstraintsFilterOptions,
std::move(*call.mutable_offer_constraints()));
if (filter.isError()) {
diff --git a/src/master/master.hpp b/src/master/master.hpp
index e478f80..3d1d472 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -2379,6 +2379,9 @@ private:
process::Time startTime; // Start time used to calculate uptime.
Option<process::Time> electedTime; // Time when this master is elected.
+
+ ::mesos::allocator::OfferConstraintsFilter::Options
+ offerConstraintsFilterOptions;
};
diff --git a/src/tests/master/offer_constraints_filter_tests.cpp b/src/tests/master/offer_constraints_filter_tests.cpp
index 84a1e91..db1976c 100644
--- a/src/tests/master/offer_constraints_filter_tests.cpp
+++ b/src/tests/master/offer_constraints_filter_tests.cpp
@@ -38,6 +38,15 @@ using ::mesos::allocator::OfferConstraintsFilter;
using ::mesos::scheduler::OfferConstraints;
+static Try<OfferConstraintsFilter> createFilter(OfferConstraints constraints)
+{
+ OfferConstraintsFilter::Options options;
+ options.re2Limits.maxMem = Kilobytes(4);
+ options.re2Limits.maxProgramSize = 100;
+ return OfferConstraintsFilter::create(options, std::move(constraints));
+}
+
+
static Try<OfferConstraints> OfferConstraintsFromJSON(const string& json)
{
Try<JSON::Object> jsonObject = JSON::parse<JSON::Object>(json);
@@ -77,8 +86,7 @@ TEST(OfferConstraintsFilter, NamedAttributeExists)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -112,8 +120,7 @@ TEST(OfferConstraintsFilter, NamedAttributeNotExists)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -147,8 +154,7 @@ TEST(OfferConstraintsFilter, NamedAttributeTextEquals)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -192,8 +198,7 @@ TEST(OfferConstraintsFilter, NamedAttributeTextNotEquals)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -238,8 +243,7 @@ TEST(OfferConstraintsFilter, TwoAttributesWithTheSameName)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -284,8 +288,7 @@ TEST(OfferConstraintsFilter, TwoConstraintsInGroup)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -332,8 +335,7 @@ TEST(OfferConstraintsFilter, TwoGroups)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
@@ -377,8 +379,7 @@ TEST(OfferConstraintsFilter, TwoRoles)
ASSERT_SOME(constraints);
- const Try<OfferConstraintsFilter> filter =
- OfferConstraintsFilter::create(std::move(*constraints));
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
ASSERT_SOME(filter);
[mesos] 05/05: Added basic tests for regexp-based constraints.
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asekretenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 355b971ead7079e3fabcb46aa0a8b9e31c9fb79e
Author: Andrei Sekretenko <as...@apache.org>
AuthorDate: Mon Aug 17 20:57:43 2020 +0200
Added basic tests for regexp-based constraints.
Review: https://reviews.apache.org/r/72787
---
.../master/offer_constraints_filter_tests.cpp | 186 +++++++++++++++++++++
1 file changed, 186 insertions(+)
diff --git a/src/tests/master/offer_constraints_filter_tests.cpp b/src/tests/master/offer_constraints_filter_tests.cpp
index db1976c..f80d56c 100644
--- a/src/tests/master/offer_constraints_filter_tests.cpp
+++ b/src/tests/master/offer_constraints_filter_tests.cpp
@@ -263,6 +263,192 @@ TEST(OfferConstraintsFilter, TwoAttributesWithTheSameName)
}
+// Tests a single TextMatches constraint on a named attribute.
+TEST(OfferConstraintsFilter, NamedAttributeTextMatches)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_matches": {"regex": "[a-d]+"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
+
+ ASSERT_SOME(filter);
+
+ // Attribute exists, is a text and matches the regex.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:abcd")));
+
+ // If an attribute is not a text, the constraint is a pass-through.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:123")));
+
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:[1-17]")));
+
+ // Attribute is a text which does not match.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:bcde")));
+
+ // Attribute does not exist.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("foo:abcd")));
+}
+
+
+// Tests a single TextNotMatches constraint on a named attribute.
+TEST(OfferConstraintsFilter, NamedAttributeTextNotMatches)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_not_matches": {"regex": "[a-d]+"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ const Try<OfferConstraintsFilter> filter = createFilter(*constraints);
+
+ ASSERT_SOME(filter);
+
+ // Attribute exists, is a text and matches the regex.
+ EXPECT_TRUE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:abcd")));
+
+ // If an attribute is not a text, the constraint is a pass-through.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:123")));
+
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:[1-17]")));
+
+ // Attribute is a text which does not match.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("bar:bcde")));
+
+ // Attribute does not exist.
+ EXPECT_FALSE(
+ filter->isAgentExcluded("roleA", slaveInfoWithAttributes("foo:abcd")));
+}
+
+
+// Tests an invalid TextMatches constraint.
+TEST(OfferConstraintsFilter, InvalidTextMatches)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_matches": {"regex": "[a-d"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ ASSERT_ERROR(createFilter(*constraints));
+}
+
+
+// Tests an invalid TextNotMatches constraint.
+TEST(OfferConstraintsFilter, InvalidTextNotMatches)
+{
+ Try<OfferConstraints> constraints = OfferConstraintsFromJSON(R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_not_matches": {"regex": "[a-d"}}
+ }]
+ }]
+ }
+ }
+ })~");
+
+ ASSERT_SOME(constraints);
+
+ ASSERT_ERROR(createFilter(*constraints));
+}
+
+
+// Tests that the constraints cannot specify a regex which will result in a too
+// complex RE2 regex program.
+TEST(OfferConstraintsFilter, RegexTooComplex)
+{
+ auto regexConstraints = [](const string& regex) {
+ return OfferConstraintsFromJSON(
+ R"~(
+ {
+ "role_constraints": {
+ "roleA": {
+ "groups": [{
+ "attribute_constraints": [{
+ "selector": {"attribute_name": "bar"},
+ "predicate": {"text_not_matches": {"regex": ")~" +
+ regex + R"~("}}
+ }]
+ }]
+ }
+ }
+ })~");
+ };
+
+ {
+ Try<OfferConstraints> good = regexConstraints("(a+){10}");
+ ASSERT_SOME(good);
+ ASSERT_SOME(createFilter(*good));
+ }
+
+ {
+ // This regexp can be compiled (fits into a memory limit) but results in
+ // a too large program.
+ Try<OfferConstraints> tooComplex = regexConstraints("(a+){50}");
+ ASSERT_SOME(tooComplex);
+
+ Try<OfferConstraintsFilter> filter = createFilter(*tooComplex);
+ ASSERT_ERROR(filter);
+ ASSERT_TRUE(strings::contains(filter.error(), "too complex"));
+ }
+
+ {
+ // This regexp does not even fit into a memory limit.
+ Try<OfferConstraints> tooLarge = regexConstraints("(a+){500}");
+ ASSERT_SOME(tooLarge);
+
+ Try<OfferConstraintsFilter> filter = createFilter(*tooLarge);
+ ASSERT_ERROR(filter);
+ ASSERT_TRUE(strings::contains(
+ filter.error(), "pattern too large - compile failed"));
+ }
+}
+
+
// Tests a single group of two constraints.
TEST(OfferConstraintsFilter, TwoConstraintsInGroup)
{