You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2017/10/11 00:29:12 UTC
[3/8] aurora git commit: Use a simpler command line argument system
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java
index 5bba496..5229450 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/HttpSecurityModule.java
@@ -14,12 +14,16 @@
package org.apache.aurora.scheduler.http.api.security;
import java.lang.reflect.Method;
+import java.util.List;
import java.util.Optional;
import java.util.Set;
import javax.servlet.Filter;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.AbstractModule;
import com.google.inject.Key;
@@ -35,11 +39,11 @@ import com.google.inject.servlet.ServletModule;
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.aurora.GuiceUtils;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
import org.apache.aurora.gen.AuroraAdmin;
import org.apache.aurora.gen.AuroraSchedulerManager;
import org.apache.aurora.scheduler.app.MoreModules;
+import org.apache.aurora.scheduler.config.CliOptions;
+import org.apache.aurora.scheduler.http.api.security.HttpSecurityModule.Options.HttpAuthenticationMechanism;
import org.apache.aurora.scheduler.thrift.aop.AnnotatedAuroraAdmin;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.guice.aop.ShiroAopModule;
@@ -73,15 +77,40 @@ public class HttpSecurityModule extends ServletModule {
private static final Key<? extends Filter> K_PERMISSIVE =
Key.get(ShiroKerberosPermissiveAuthenticationFilter.class);
- @CmdLine(name = "shiro_realm_modules",
- help = "Guice modules for configuring Shiro Realms.")
- private static final Arg<Set<Module>> SHIRO_REALM_MODULE = Arg.create(
- ImmutableSet.of(MoreModules.lazilyInstantiated(IniShiroRealmModule.class)));
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-shiro_realm_modules",
+ description = "Guice modules for configuring Shiro Realms.")
+ @SuppressWarnings("rawtypes")
+ public List<Class> shiroRealmModule = ImmutableList.of(IniShiroRealmModule.class);
+
+ @Parameter(names = "-shiro_after_auth_filter",
+ description = "Fully qualified class name of the servlet filter to be applied after the"
+ + " shiro auth filters are applied.")
+ public Class<? extends Filter> shiroAfterAuthFilter;
+
+ public enum HttpAuthenticationMechanism {
+ /**
+ * No security.
+ */
+ NONE,
+
+ /**
+ * HTTP Basic Authentication, produces {@link org.apache.shiro.authc.UsernamePasswordToken}s.
+ */
+ BASIC,
+
+ /**
+ * Use GSS-Negotiate. Only Kerberos and SPNEGO-with-Kerberos GSS mechanisms are supported.
+ */
+ NEGOTIATE,
+ }
- @CmdLine(name = "shiro_after_auth_filter",
- help = "Fully qualified class name of the servlet filter to be applied after the"
- + " shiro auth filters are applied.")
- private static final Arg<Class<? extends Filter>> SHIRO_AFTER_AUTH_FILTER = Arg.create();
+ @Parameter(names = "-http_authentication_mechanism",
+ description = "HTTP Authentication mechanism to use.")
+ public HttpAuthenticationMechanism httpAuthenticationMechanism =
+ HttpAuthenticationMechanism.NONE;
+ }
@VisibleForTesting
static final Matcher<Method> AURORA_SCHEDULER_MANAGER_SERVICE =
@@ -91,36 +120,15 @@ public class HttpSecurityModule extends ServletModule {
static final Matcher<Method> AURORA_ADMIN_SERVICE =
GuiceUtils.interfaceMatcher(AuroraAdmin.Iface.class, true);
- public enum HttpAuthenticationMechanism {
- /**
- * No security.
- */
- NONE,
-
- /**
- * HTTP Basic Authentication, produces {@link org.apache.shiro.authc.UsernamePasswordToken}s.
- */
- BASIC,
-
- /**
- * Use GSS-Negotiate. Only Kerberos and SPNEGO-with-Kerberos GSS mechanisms are supported.
- */
- NEGOTIATE,
- }
-
- @CmdLine(name = "http_authentication_mechanism", help = "HTTP Authentication mechanism to use.")
- private static final Arg<HttpAuthenticationMechanism> HTTP_AUTHENTICATION_MECHANISM =
- Arg.create(HttpAuthenticationMechanism.NONE);
-
private final HttpAuthenticationMechanism mechanism;
private final Set<Module> shiroConfigurationModules;
private final Optional<Key<? extends Filter>> shiroAfterAuthFilterKey;
- public HttpSecurityModule() {
+ public HttpSecurityModule(CliOptions options) {
this(
- HTTP_AUTHENTICATION_MECHANISM.get(),
- SHIRO_REALM_MODULE.get(),
- SHIRO_AFTER_AUTH_FILTER.hasAppliedValue() ? Key.get(SHIRO_AFTER_AUTH_FILTER.get()) : null);
+ options.httpSecurity.httpAuthenticationMechanism,
+ MoreModules.instantiateAll(options.httpSecurity.shiroRealmModule, options),
+ Optional.ofNullable(options.httpSecurity.shiroAfterAuthFilter).map(Key::get).orElse(null));
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java
index 9458468..fc4c3ec 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/IniShiroRealmModule.java
@@ -15,13 +15,14 @@ package org.apache.aurora.scheduler.http.api.security;
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
+import org.apache.aurora.scheduler.config.CliOptions;
import org.apache.shiro.authc.credential.CredentialsMatcher;
import org.apache.shiro.authc.credential.SimpleCredentialsMatcher;
import org.apache.shiro.config.Ini;
@@ -38,21 +39,27 @@ import org.apache.shiro.realm.text.IniRealm;
* used to provide authorization configuration and the passwords will be ignored.
*/
public class IniShiroRealmModule extends AbstractModule {
- @CmdLine(name = "shiro_ini_path",
- help = "Path to shiro.ini for authentication and authorization configuration.")
- private static final Arg<Ini> SHIRO_INI_PATH = Arg.create(null);
- @CmdLine(name = "shiro_credentials_matcher",
- help = "The shiro credentials matcher to use (will be constructed by Guice).")
- private static final Arg<Class<? extends CredentialsMatcher>> SHIRO_CREDENTIALS_MATCHER =
- Arg.<Class<? extends CredentialsMatcher>>create(SimpleCredentialsMatcher.class);
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-shiro_ini_path",
+ description = "Path to shiro.ini for authentication and authorization configuration.",
+ converter = ShiroIniConverter.class)
+ public Ini shiroIniPath;
+
+ @Parameter(names = "-shiro_credentials_matcher",
+ description = "The shiro credentials matcher to use (will be constructed by Guice).")
+ public Class<? extends CredentialsMatcher> shiroCredentialsMatcher =
+ SimpleCredentialsMatcher.class;
+ }
private final Optional<Ini> ini;
private final Optional<Class<? extends CredentialsMatcher>> shiroCredentialsMatcher;
- public IniShiroRealmModule() {
- this(Optional.fromNullable(SHIRO_INI_PATH.get()),
- Optional.fromNullable(SHIRO_CREDENTIALS_MATCHER.get()));
+ public IniShiroRealmModule(CliOptions options) {
+ this(
+ Optional.fromNullable(options.iniShiroRealm.shiroIniPath),
+ Optional.fromNullable(options.iniShiroRealm.shiroCredentialsMatcher));
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java
index 9c7aead..4a03798 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/Kerberos5ShiroRealmModule.java
@@ -24,6 +24,8 @@ import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.io.Files;
@@ -32,8 +34,7 @@ import com.google.inject.PrivateModule;
import com.sun.security.auth.login.ConfigFile;
import com.sun.security.auth.module.Krb5LoginModule;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
+import org.apache.aurora.scheduler.config.CliOptions;
import org.ietf.jgss.GSSCredential;
import org.ietf.jgss.GSSException;
import org.ietf.jgss.GSSManager;
@@ -59,9 +60,6 @@ public class Kerberos5ShiroRealmModule extends AbstractModule {
*/
private static final String GSS_SPNEGO_MECH_OID = "1.3.6.1.5.5.2";
- private static final String SERVER_KEYTAB_ARGNAME = "kerberos_server_keytab";
- private static final String SERVER_PRINCIPAL_ARGNAME = "kerberos_server_principal";
-
private static final String JAAS_CONF_TEMPLATE =
"%s {\n"
+ Krb5LoginModule.class.getName()
@@ -69,28 +67,36 @@ public class Kerberos5ShiroRealmModule extends AbstractModule {
+ "keyTab=\"%s\" principal=\"%s\" debug=%s;\n"
+ "};";
- @CmdLine(name = SERVER_KEYTAB_ARGNAME, help = "Path to the server keytab.")
- private static final Arg<File> SERVER_KEYTAB = Arg.create(null);
+ @Parameters(separators = "=")
+ public static class Options {
+ private static final String SERVER_KEYTAB_ARGNAME = "-kerberos_server_keytab";
+ private static final String SERVER_PRINCIPAL_ARGNAME = "-kerberos_server_principal";
+
+ @Parameter(names = SERVER_KEYTAB_ARGNAME, description = "Path to the server keytab.")
+ public File serverKeytab;
- @CmdLine(name = SERVER_PRINCIPAL_ARGNAME,
- help = "Kerberos server principal to use, usually of the form "
- + "HTTP/aurora.example.com@EXAMPLE.COM")
- private static final Arg<KerberosPrincipal> SERVER_PRINCIPAL = Arg.create(null);
+ @Parameter(names = SERVER_PRINCIPAL_ARGNAME,
+ description = "Kerberos server principal to use, usually of the form "
+ + "HTTP/aurora.example.com@EXAMPLE.COM")
+ public KerberosPrincipal serverPrincipal;
- @CmdLine(name = "kerberos_debug", help = "Produce additional Kerberos debugging output.")
- private static final Arg<Boolean> DEBUG = Arg.create(false);
+ @Parameter(names = "-kerberos_debug",
+ description = "Produce additional Kerberos debugging output.",
+ arity = 1)
+ public boolean kerberosDebug = false;
+ }
private final Optional<File> serverKeyTab;
private final Optional<KerberosPrincipal> serverPrincipal;
private final GSSManager gssManager;
private final boolean kerberosDebugEnabled;
- public Kerberos5ShiroRealmModule() {
+ public Kerberos5ShiroRealmModule(CliOptions options) {
this(
- Optional.fromNullable(SERVER_KEYTAB.get()),
- Optional.fromNullable(SERVER_PRINCIPAL.get()),
+ Optional.fromNullable(options.kerberos.serverKeytab),
+ Optional.fromNullable(options.kerberos.serverPrincipal),
GSSManager.getInstance(),
- DEBUG.get());
+ options.kerberos.kerberosDebug);
}
@VisibleForTesting
@@ -121,12 +127,12 @@ public class Kerberos5ShiroRealmModule extends AbstractModule {
@Override
protected void configure() {
if (!serverKeyTab.isPresent()) {
- addError("No -" + SERVER_KEYTAB_ARGNAME + " specified.");
+ addError("No -" + Options.SERVER_KEYTAB_ARGNAME + " specified.");
return;
}
if (!serverPrincipal.isPresent()) {
- addError("No -" + SERVER_PRINCIPAL_ARGNAME + " specified.");
+ addError("No -" + Options.SERVER_PRINCIPAL_ARGNAME + " specified.");
return;
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalConverter.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalConverter.java
new file mode 100644
index 0000000..2c9ea31
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalConverter.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http.api.security;
+
+import javax.security.auth.kerberos.KerberosPrincipal;
+
+import com.beust.jcommander.ParameterException;
+import com.beust.jcommander.converters.BaseConverter;
+
+public class KerberosPrincipalConverter extends BaseConverter<KerberosPrincipal> {
+
+ public KerberosPrincipalConverter() {
+ super("");
+ }
+
+ public KerberosPrincipalConverter(String optionName) {
+ super(optionName);
+ }
+
+ @Override
+ public KerberosPrincipal convert(String value) {
+ try {
+ return new KerberosPrincipal(value);
+ } catch (IllegalArgumentException e) {
+ throw new ParameterException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalParser.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalParser.java
deleted file mode 100644
index bce20b7..0000000
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/KerberosPrincipalParser.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.http.api.security;
-
-import javax.security.auth.kerberos.KerberosPrincipal;
-
-import org.apache.aurora.common.args.ArgParser;
-import org.apache.aurora.common.args.parsers.NonParameterizedTypeParser;
-
-@ArgParser
-class KerberosPrincipalParser extends NonParameterizedTypeParser<KerberosPrincipal> {
- @Override
- public KerberosPrincipal doParse(String raw) throws IllegalArgumentException {
- return new KerberosPrincipal(raw);
- }
-}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/ModuleParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/ModuleParser.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/ModuleParser.java
deleted file mode 100644
index ccd9a20..0000000
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/ModuleParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.http.api.security;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.Module;
-
-import org.apache.aurora.common.args.ArgParser;
-import org.apache.aurora.common.args.parsers.NonParameterizedTypeParser;
-import org.apache.aurora.scheduler.app.MoreModules;
-
-/**
- * ArgParser for Guice modules. Constructs an instance of a Module with a given alias or FQCN if it
- * has a public no-args constructor.
- */
-@ArgParser
-public class ModuleParser extends NonParameterizedTypeParser<Module> {
- private static final Map<String, String> NAME_ALIASES = ImmutableMap.of(
- "KERBEROS5_AUTHN", Kerberos5ShiroRealmModule.class.getCanonicalName(),
- "INI_AUTHNZ", IniShiroRealmModule.class.getCanonicalName());
-
- @Override
- public Module doParse(String raw) throws IllegalArgumentException {
- String fullyQualifiedName = NAME_ALIASES.containsKey(raw) ? NAME_ALIASES.get(raw) : raw;
- Class<?> rawClass;
- try {
- rawClass = Class.forName(fullyQualifiedName);
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException(e);
- }
-
- if (!Module.class.isAssignableFrom(rawClass)) {
- throw new IllegalArgumentException(
- "Class " + fullyQualifiedName + " must implement " + Module.class.getName());
- }
- @SuppressWarnings("unchecked")
- Class<? extends Module> moduleClass = (Class<? extends Module>) rawClass;
-
- return MoreModules.lazilyInstantiated(moduleClass);
- }
-}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniConverter.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniConverter.java
new file mode 100644
index 0000000..3f8f777
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniConverter.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http.api.security;
+
+import java.util.Set;
+
+import com.beust.jcommander.ParameterException;
+import com.beust.jcommander.converters.BaseConverter;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Sets;
+
+import org.apache.shiro.config.ConfigurationException;
+import org.apache.shiro.config.Ini;
+import org.apache.shiro.realm.text.IniRealm;
+
+/**
+ * Parser for shiro.ini files. Accepts any string that {@link Ini#fromResourcePath(String)} does.
+ * The provided ini file may have only the sections required for configuration
+ * ({@link IniRealm#ROLES_SECTION_NAME} and {@link IniRealm#USERS_SECTION_NAME}) and no extras -
+ * Aurora uses Guice in to configure those sections in {@link HttpSecurityModule}}.
+ */
+public class ShiroIniConverter extends BaseConverter<Ini> {
+ @VisibleForTesting
+ static final Set<String> ALLOWED_SECTION_NAMES =
+ ImmutableSortedSet.of(IniRealm.ROLES_SECTION_NAME, IniRealm.USERS_SECTION_NAME);
+
+ public ShiroIniConverter() {
+ super("");
+ }
+
+ public ShiroIniConverter(String optionName) {
+ super(optionName);
+ }
+
+ @VisibleForTesting
+ static class ExtraSectionsException extends IllegalArgumentException {
+ ExtraSectionsException(Set<String> extraSections) {
+ super("Extra sections present: " + extraSections);
+ }
+ }
+
+ @VisibleForTesting
+ static class MissingSectionsException extends ParameterException {
+ MissingSectionsException() {
+ super("No sections present. Allowed sections are: "
+ + Joiner.on(",").join(ALLOWED_SECTION_NAMES));
+ }
+ }
+
+ @VisibleForTesting
+ static class ShiroConfigurationException extends ParameterException {
+ ShiroConfigurationException(ConfigurationException e) {
+ super(e);
+ }
+ }
+
+ @Override
+ public Ini convert(String raw) {
+ Ini ini;
+ try {
+ ini = Ini.fromResourcePath(raw);
+ } catch (ConfigurationException e) {
+ throw new ParameterException(getErrorString(raw, e.getMessage()), e);
+ }
+
+ Set<String> presentSections = ImmutableSortedSet.copyOf(ini.getSectionNames());
+ if (presentSections.isEmpty()) {
+ throw new MissingSectionsException();
+ }
+
+ Set<String> extraSections = Sets.difference(presentSections, ALLOWED_SECTION_NAMES);
+ if (!extraSections.isEmpty()) {
+ throw new ExtraSectionsException(extraSections);
+ }
+
+ return ini;
+ }
+}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniParser.java b/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniParser.java
deleted file mode 100644
index 3ee41b8..0000000
--- a/src/main/java/org/apache/aurora/scheduler/http/api/security/ShiroIniParser.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.http.api.security;
-
-import java.util.Set;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Sets;
-
-import org.apache.aurora.common.args.ArgParser;
-import org.apache.aurora.common.args.parsers.NonParameterizedTypeParser;
-import org.apache.shiro.config.ConfigurationException;
-import org.apache.shiro.config.Ini;
-import org.apache.shiro.realm.text.IniRealm;
-
-/**
- * Parser for shiro.ini files. Accepts any string that {@link Ini#fromResourcePath(String)} does.
- * The provided ini file may have only the sections required for configuration
- * ({@link IniRealm.ROLES_SECTION_NAME} and {@link IniRealm.USERS_SECTION_NAME}) and no extras -
- * Aurora uses Guice in to configure those sections in {@link HttpSecurityModule}}.
- */
-@ArgParser
-public class ShiroIniParser extends NonParameterizedTypeParser<Ini> {
- @VisibleForTesting
- static final Set<String> ALLOWED_SECTION_NAMES =
- ImmutableSortedSet.of(IniRealm.ROLES_SECTION_NAME, IniRealm.USERS_SECTION_NAME);
-
- @VisibleForTesting
- static class ExtraSectionsException extends IllegalArgumentException {
- ExtraSectionsException(Set<String> extraSections) {
- super("Extra sections present: " + extraSections);
- }
- }
-
- @VisibleForTesting
- static class MissingSectionsException extends IllegalArgumentException {
- MissingSectionsException() {
- super("No sections present. Allowed sections are: "
- + Joiner.on(",").join(ALLOWED_SECTION_NAMES));
- }
- }
-
- @VisibleForTesting
- static class ShiroConfigurationException extends IllegalArgumentException {
- ShiroConfigurationException(ConfigurationException e) {
- super(e);
- }
- }
-
- @Override
- public Ini doParse(String raw) throws IllegalArgumentException {
- Ini ini;
- try {
- ini = Ini.fromResourcePath(raw);
- } catch (ConfigurationException e) {
- throw new ShiroConfigurationException(e);
- }
-
- Set<String> presentSections = ImmutableSortedSet.copyOf(ini.getSectionNames());
- if (presentSections.isEmpty()) {
- throw new MissingSectionsException();
- }
-
- Set<String> extraSections = Sets.difference(presentSections, ALLOWED_SECTION_NAMES);
- if (!extraSections.isEmpty()) {
- throw new ExtraSectionsException(extraSections);
- }
-
- return ini;
- }
-}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
index 9a6c0c4..a4984a9 100644
--- a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java
@@ -16,12 +16,13 @@ package org.apache.aurora.scheduler.log.mesos;
import java.io.File;
import java.net.InetSocketAddress;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.inject.PrivateModule;
@@ -29,13 +30,12 @@ import com.google.inject.Provides;
import com.google.inject.TypeLiteral;
import org.apache.aurora.codec.ThriftBinaryCodec;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
import org.apache.aurora.common.net.InetSocketAddressHelper;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.common.zookeeper.Credentials;
import org.apache.aurora.gen.storage.LogEntry;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
import org.apache.aurora.scheduler.discovery.ServiceDiscoveryBindings;
import org.apache.aurora.scheduler.discovery.ZooKeeperConfig;
import org.apache.aurora.scheduler.log.mesos.LogInterface.ReaderInterface;
@@ -52,86 +52,75 @@ import org.apache.zookeeper.common.PathUtils;
* </ul>
*/
public class MesosLogStreamModule extends PrivateModule {
- @CmdLine(name = "native_log_quorum_size",
- help = "The size of the quorum required for all log mutations.")
- private static final Arg<Integer> QUORUM_SIZE = Arg.create(1);
-
- @CmdLine(name = "native_log_file_path",
- help = "Path to a file to store the native log data in. If the parent directory does"
- + "not exist it will be created.")
- private static final Arg<File> LOG_PATH = Arg.create(null);
-
- @CmdLine(name = "native_log_zk_group_path",
- help = "A zookeeper node for use by the native log to track the master coordinator.")
- private static final Arg<String> ZK_LOG_GROUP_PATH = Arg.create(null);
-
- /*
- * This timeout includes the time to get a quorum to promise leadership to the coordinator and
- * the time to fill any holes in the coordinator's log.
- */
- @CmdLine(name = "native_log_election_timeout",
- help = "The timeout for a single attempt to obtain a new log writer.")
- private static final Arg<Amount<Long, Time>> COORDINATOR_ELECTION_TIMEOUT =
- Arg.create(Amount.of(15L, Time.SECONDS));
-
- /*
- * Normally retries would not be expected to help much - however in the small replica set where
- * a few down replicas doom a coordinator election attempt, retrying effectively gives us a wider
- * window in which to await a live quorum before giving up and thrashing the global election
- * process. Observed log replica recovery times as of 4/6/2012 can be ~45 seconds so giving a
- * window >= 2x this should support 1-round election events (that possibly use several retries in
- * the single round).
- */
- @CmdLine(name = "native_log_election_retries",
- help = "The maximum number of attempts to obtain a new log writer.")
- private static final Arg<Integer> COORDINATOR_ELECTION_RETRIES = Arg.create(20);
-
- @CmdLine(name = "native_log_read_timeout",
- help = "The timeout for doing log reads.")
- private static final Arg<Amount<Long, Time>> READ_TIMEOUT =
- Arg.create(Amount.of(5L, Time.SECONDS));
-
- @CmdLine(name = "native_log_write_timeout",
- help = "The timeout for doing log appends and truncations.")
- private static final Arg<Amount<Long, Time>> WRITE_TIMEOUT =
- Arg.create(Amount.of(3L, Time.SECONDS));
-
- private static <T> T getRequiredArg(Arg<T> arg, String name) {
- if (!arg.hasAppliedValue()) {
- throw new IllegalStateException(
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-native_log_quorum_size",
+ description = "The size of the quorum required for all log mutations.")
+ public int quorumSize = 1;
+
+ @Parameter(names = "-native_log_file_path",
+ description =
+ "Path to a file to store the native log data in. If the parent directory does"
+ + "not exist it will be created.")
+ public File logPath = null;
+
+ @Parameter(names = "-native_log_zk_group_path",
+ description = "A zookeeper node for use by the native log to track the master coordinator.")
+ public String zkLogGroupPath = null;
+
+ /*
+ * This timeout includes the time to get a quorum to promise leadership to the coordinator and
+ * the time to fill any holes in the coordinator's log.
+ */
+ @Parameter(names = "-native_log_election_timeout",
+ description = "The timeout for a single attempt to obtain a new log writer.")
+ public TimeAmount coordinatorElectionTimeout = new TimeAmount(15, Time.SECONDS);
+
+ /**
+ * Normally retries would not be expected to help much - however in the small replica set where
+ * a few down replicas doom a coordinator election attempt, retrying effectively gives us a
+ * wider window in which to await a live quorum before giving up and thrashing the global
+ * election process. Observed log replica recovery times as of 4/6/2012 can be ~45 seconds so
+ * giving a window >= 2x this should support 1-round election events (that possibly use several
+ * retries in the single round).
+ */
+ @Parameter(names = "-native_log_election_retries",
+ description = "The maximum number of attempts to obtain a new log writer.")
+ public int coordinatorElectionRetries = 20;
+
+ @Parameter(names = "-native_log_read_timeout",
+ description = "The timeout for doing log reads.")
+ public TimeAmount readTimeout = new TimeAmount(5, Time.SECONDS);
+
+ @Parameter(names = "-native_log_write_timeout",
+ description = "The timeout for doing log appends and truncations.")
+ public TimeAmount writeTimeout = new TimeAmount(3, Time.SECONDS);
+ }
+
+ private static void requireArg(Object arg, String name) {
+ if (arg == null) {
+ throw new IllegalArgumentException(
String.format("A value for the -%s flag must be supplied", name));
}
- return arg.get();
}
+ private final Options options;
private final ZooKeeperConfig zkClientConfig;
- private final File logPath;
- private final String zkLogGroupPath;
-
- public MesosLogStreamModule(ZooKeeperConfig zkClientConfig) {
- this(zkClientConfig,
- getRequiredArg(LOG_PATH, "native_log_file_path"),
- getRequiredArg(ZK_LOG_GROUP_PATH, "native_log_zk_group_path"));
- }
-
- public MesosLogStreamModule(
- ZooKeeperConfig zkClientConfig,
- File logPath,
- String zkLogGroupPath) {
-
- this.zkClientConfig = Objects.requireNonNull(zkClientConfig);
- this.logPath = Objects.requireNonNull(logPath);
- PathUtils.validatePath(zkLogGroupPath); // This checks for null.
- this.zkLogGroupPath = zkLogGroupPath;
+ public MesosLogStreamModule(Options options, ZooKeeperConfig zkClientConfig) {
+ this.options = options;
+ requireArg(options.logPath, "native_log_file_path");
+ requireArg(options.zkLogGroupPath, "native_log_zk_group_path");
+ PathUtils.validatePath(options.zkLogGroupPath);
+ this.zkClientConfig = zkClientConfig;
}
@Override
protected void configure() {
bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.ReadTimeout.class)
- .toInstance(READ_TIMEOUT.get());
+ .toInstance(options.readTimeout);
bind(new TypeLiteral<Amount<Long, Time>>() { }).annotatedWith(MesosLog.WriteTimeout.class)
- .toInstance(WRITE_TIMEOUT.get());
+ .toInstance(options.writeTimeout);
bind(org.apache.aurora.scheduler.log.Log.class).to(MesosLog.class);
bind(MesosLog.class).in(Singleton.class);
@@ -141,7 +130,7 @@ public class MesosLogStreamModule extends PrivateModule {
@Provides
@Singleton
Log provideLog(@ServiceDiscoveryBindings.ZooKeeper Iterable<InetSocketAddress> servers) {
- File parentDir = logPath.getParentFile();
+ File parentDir = options.logPath.getParentFile();
if (!parentDir.exists() && !parentDir.mkdirs()) {
addError("Failed to create parent directory to store native log at: %s", parentDir);
}
@@ -152,22 +141,22 @@ public class MesosLogStreamModule extends PrivateModule {
if (zkClientConfig.getCredentials().isPresent()) {
Credentials zkCredentials = zkClientConfig.getCredentials().get();
return new Log(
- QUORUM_SIZE.get(),
- logPath.getAbsolutePath(),
+ options.quorumSize,
+ options.logPath.getAbsolutePath(),
zkConnectString,
zkClientConfig.getSessionTimeout().getValue(),
zkClientConfig.getSessionTimeout().getUnit().getTimeUnit(),
- zkLogGroupPath,
+ options.zkLogGroupPath,
zkCredentials.scheme(),
zkCredentials.authToken());
} else {
return new Log(
- QUORUM_SIZE.get(),
- logPath.getAbsolutePath(),
+ options.quorumSize,
+ options.logPath.getAbsolutePath(),
zkConnectString,
zkClientConfig.getSessionTimeout().getValue(),
zkClientConfig.getSessionTimeout().getUnit().getTimeUnit(),
- zkLogGroupPath);
+ options.zkLogGroupPath);
}
}
@@ -178,9 +167,12 @@ public class MesosLogStreamModule extends PrivateModule {
@Provides
Log.Writer provideWriter(Log log) {
- Amount<Long, Time> electionTimeout = COORDINATOR_ELECTION_TIMEOUT.get();
- return new Log.Writer(log, electionTimeout.getValue(), electionTimeout.getUnit().getTimeUnit(),
- COORDINATOR_ELECTION_RETRIES.get());
+ Amount<Long, Time> electionTimeout = options.coordinatorElectionTimeout;
+ return new Log.Writer(
+ log,
+ electionTimeout.getValue(),
+ electionTimeout.getUnit().getTimeUnit(),
+ options.coordinatorElectionRetries);
}
@Provides
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java b/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java
index 97d3c20..5e83b2a 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/CommandLineDriverSettingsModule.java
@@ -19,18 +19,19 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
+
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.AbstractModule;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.common.args.constraints.NotNull;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
import org.apache.mesos.v1.Protos;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,85 +49,88 @@ public class CommandLineDriverSettingsModule extends AbstractModule {
private static final Logger LOG =
LoggerFactory.getLogger(CommandLineDriverSettingsModule.class);
- @NotNull
- @CmdLine(name = "mesos_master_address",
- help = "Address for the mesos master, can be a socket address or zookeeper path.")
- private static final Arg<String> MESOS_MASTER_ADDRESS = Arg.create();
-
- @VisibleForTesting
- static final String PRINCIPAL_KEY = "aurora_authentication_principal";
- @VisibleForTesting
- static final String SECRET_KEY = "aurora_authentication_secret";
-
- @CmdLine(name = "framework_authentication_file",
- help = "Properties file which contains framework credentials to authenticate with Mesos"
- + "master. Must contain the properties '" + PRINCIPAL_KEY + "' and "
- + "'" + SECRET_KEY + "'.")
- private static final Arg<File> FRAMEWORK_AUTHENTICATION_FILE = Arg.create();
-
- @CmdLine(name = "framework_failover_timeout",
- help = "Time after which a framework is considered deleted. SHOULD BE VERY HIGH.")
- private static final Arg<Amount<Long, Time>> FRAMEWORK_FAILOVER_TIMEOUT =
- Arg.create(Amount.of(21L, Time.DAYS));
-
- @CmdLine(name = "framework_announce_principal",
- help = "When 'framework_authentication_file' flag is set, the FrameworkInfo "
- + "registered with the mesos master will also contain the principal. This is "
- + "necessary if you intend to use mesos authorization via mesos ACLs. "
- + "The default will change in a future release. Changing this value is backwards "
- + "incompatible. For details, see MESOS-703.")
- private static final Arg<Boolean> FRAMEWORK_ANNOUNCE_PRINCIPAL = Arg.create(false);
-
- @CmdLine(name = "framework_name",
- help = "Name used to register the Aurora framework with Mesos.")
- private static final Arg<String> FRAMEWORK_NAME = Arg.create("Aurora");
-
- @CmdLine(name = "executor_user",
- help = "User to start the executor. Defaults to \"root\". "
- + "Set this to an unprivileged user if the mesos master was started with "
- + "\"--no-root_submissions\". If set to anything other than \"root\", the executor "
- + "will ignore the \"role\" setting for jobs since it can't use setuid() anymore. "
- + "This means that all your jobs will run under the specified user and the user has "
- + "to exist on the Mesos agents.")
- private static final Arg<String> EXECUTOR_USER = Arg.create("root");
-
- @CmdLine(name = "receive_revocable_resources",
- help = "Allows receiving revocable resource offers from Mesos.")
- private static final Arg<Boolean> RECEIVE_REVOCABLE_RESOURCES = Arg.create(false);
-
- @CmdLine(name = "mesos_role",
- help = "The Mesos role this framework will register as. The default is to left this empty, "
- + "and the framework will register without any role and only receive unreserved "
- + "resources in offer.")
- private static final Arg<String> MESOS_ROLE = Arg.create();
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-mesos_master_address",
+ required = true,
+ description = "Address for the mesos master, can be a socket address or zookeeper path.")
+ public String mesosMasterAddress;
+
+ public static final String PRINCIPAL_KEY = "aurora_authentication_principal";
+ public static final String SECRET_KEY = "aurora_authentication_secret";
+
+ @Parameter(names = "-framework_authentication_file",
+ description =
+ "Properties file which contains framework credentials to authenticate with Mesos"
+ + "master. Must contain the properties '" + PRINCIPAL_KEY + "' and "
+ + "'" + SECRET_KEY + "'.")
+ public File frameworkAuthenticationFile;
+
+ @Parameter(names = "-framework_failover_timeout",
+ description = "Time after which a framework is considered deleted. SHOULD BE VERY HIGH.")
+ public TimeAmount frameworkFailoverTimeout = new TimeAmount(21, Time.DAYS);
+
+ @Parameter(names = "-framework_announce_principal",
+ description = "When 'framework_authentication_file' flag is set, the FrameworkInfo "
+ + "registered with the mesos master will also contain the principal. This is "
+ + "necessary if you intend to use mesos authorization via mesos ACLs. "
+ + "The default will change in a future release. Changing this value is backwards "
+ + "incompatible. For details, see MESOS-703.",
+ arity = 1)
+ public boolean frameworkAnnouncePrincipal = false;
+
+ @Parameter(names = "-framework_name",
+ description = "Name used to register the Aurora framework with Mesos.")
+ public String frameworkName = "Aurora";
+
+ @Parameter(names = "-executor_user",
+ description = "User to start the executor. Defaults to \"root\". "
+ + "Set this to an unprivileged user if the mesos master was started with "
+ + "\"--no-root_submissions\". If set to anything other than \"root\", the executor "
+ + "will ignore the \"role\" setting for jobs since it can't use setuid() anymore. "
+ + "This means that all your jobs will run under the specified user and the user has "
+ + "to exist on the Mesos agents.")
+ public String executorUser = "root";
+
+ @Parameter(names = "-receive_revocable_resources",
+ description = "Allows receiving revocable resource offers from Mesos.",
+ arity = 1)
+ public boolean receiveRevocableResources = false;
+
+ @Parameter(names = "-mesos_role",
+ description =
+ "The Mesos role this framework will register as. The default is to left this empty, "
+ + "and the framework will register without any role and only receive unreserved "
+ + "resources in offer.")
+ public String mesosRole;
+ }
+ private final Options options;
private final boolean allowGpuResource;
- public CommandLineDriverSettingsModule(boolean allowGpuResource) {
+ public CommandLineDriverSettingsModule(Options options, boolean allowGpuResource) {
+ this.options = options;
this.allowGpuResource = allowGpuResource;
}
@Override
protected void configure() {
- Optional<Protos.Credential> credentials = getCredentials();
+ Optional<Protos.Credential> credentials = getCredentials(options);
Optional<String> principal = Optional.absent();
- if (FRAMEWORK_ANNOUNCE_PRINCIPAL.get() && credentials.isPresent()) {
+ if (options.frameworkAnnouncePrincipal && credentials.isPresent()) {
principal = Optional.of(credentials.get().getPrincipal());
}
- Optional<String> role =
- MESOS_ROLE.hasAppliedValue() ? Optional.of(MESOS_ROLE.get()) : Optional.absent();
- DriverSettings settings = new DriverSettings(
- MESOS_MASTER_ADDRESS.get(),
- credentials);
+ Optional<String> role = Optional.fromNullable(options.mesosRole);
+ DriverSettings settings = new DriverSettings(options.mesosMasterAddress, credentials);
bind(DriverSettings.class).toInstance(settings);
FrameworkInfo base =
buildFrameworkInfo(
- FRAMEWORK_NAME.get(),
- EXECUTOR_USER.get(),
+ options.frameworkName,
+ options.executorUser,
principal,
- FRAMEWORK_FAILOVER_TIMEOUT.get(),
- RECEIVE_REVOCABLE_RESOURCES.get(),
+ options.frameworkFailoverTimeout,
+ options.receiveRevocableResources,
allowGpuResource,
role);
bind(FrameworkInfo.class)
@@ -137,11 +141,13 @@ public class CommandLineDriverSettingsModule extends AbstractModule {
}
- private static Optional<Protos.Credential> getCredentials() {
- if (FRAMEWORK_AUTHENTICATION_FILE.hasAppliedValue()) {
+ private static Optional<Protos.Credential> getCredentials(Options opts) {
+ if (opts.frameworkAuthenticationFile == null) {
+ return Optional.absent();
+ } else {
Properties properties;
try {
- properties = parseCredentials(new FileInputStream(FRAMEWORK_AUTHENTICATION_FILE.get()));
+ properties = parseCredentials(new FileInputStream(opts.frameworkAuthenticationFile));
} catch (FileNotFoundException e) {
LOG.error("Authentication File not Found");
throw new RuntimeException(e);
@@ -149,14 +155,12 @@ public class CommandLineDriverSettingsModule extends AbstractModule {
LOG.info(
"Connecting to master using authentication (principal: {}).",
- properties.get(PRINCIPAL_KEY));
+ properties.get(Options.PRINCIPAL_KEY));
return Optional.of(Protos.Credential.newBuilder()
- .setPrincipal(properties.getProperty(PRINCIPAL_KEY))
- .setSecret(properties.getProperty(SECRET_KEY))
+ .setPrincipal(properties.getProperty(Options.PRINCIPAL_KEY))
+ .setSecret(properties.getProperty(Options.SECRET_KEY))
.build());
- } else {
- return Optional.absent();
}
}
@@ -208,10 +212,10 @@ public class CommandLineDriverSettingsModule extends AbstractModule {
LOG.error("Unable to load authentication file");
throw new RuntimeException(e);
}
- Preconditions.checkState(properties.containsKey(PRINCIPAL_KEY),
- "The framework authentication file is missing the key: %s", PRINCIPAL_KEY);
- Preconditions.checkState(properties.containsKey(SECRET_KEY),
- "The framework authentication file is missing the key: %s", SECRET_KEY);
+ Preconditions.checkState(properties.containsKey(Options.PRINCIPAL_KEY),
+ "The framework authentication file is missing the key: %s", Options.PRINCIPAL_KEY);
+ Preconditions.checkState(properties.containsKey(Options.SECRET_KEY),
+ "The framework authentication file is missing the key: %s", Options.SECRET_KEY);
return properties;
}
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/mesos/LibMesosLoadingModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/LibMesosLoadingModule.java b/src/main/java/org/apache/aurora/scheduler/mesos/LibMesosLoadingModule.java
index 3e943ff..ec07279 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/LibMesosLoadingModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/LibMesosLoadingModule.java
@@ -15,7 +15,7 @@ package org.apache.aurora.scheduler.mesos;
import com.google.inject.AbstractModule;
-import org.apache.aurora.scheduler.app.SchedulerMain;
+import org.apache.aurora.scheduler.app.SchedulerMain.Options.DriverKind;
import org.apache.mesos.v1.scheduler.V0Mesos;
import org.apache.mesos.v1.scheduler.V1Mesos;
@@ -25,9 +25,9 @@ import static com.google.common.base.Preconditions.checkState;
* A module that binds a driver factory which requires the libmesos native libary.
*/
public class LibMesosLoadingModule extends AbstractModule {
- private final SchedulerMain.DriverKind kind;
+ private final DriverKind kind;
- public LibMesosLoadingModule(SchedulerMain.DriverKind kind) {
+ public LibMesosLoadingModule(DriverKind kind) {
this.kind = kind;
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverModule.java b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverModule.java
index 18dc3e0..b54e1f3 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverModule.java
@@ -23,7 +23,7 @@ import javax.inject.Singleton;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.AbstractModule;
-import org.apache.aurora.scheduler.app.SchedulerMain;
+import org.apache.aurora.scheduler.app.SchedulerMain.Options.DriverKind;
import org.apache.aurora.scheduler.base.AsyncUtil;
import org.apache.aurora.scheduler.events.PubsubEventModule;
import org.apache.aurora.scheduler.mesos.MesosCallbackHandler.MesosCallbackHandlerImpl;
@@ -43,9 +43,9 @@ import static com.google.common.base.Preconditions.checkState;
*/
public class SchedulerDriverModule extends AbstractModule {
private static final Logger LOG = LoggerFactory.getLogger(SchedulerDriverModule.class);
- private final SchedulerMain.DriverKind kind;
+ private final DriverKind kind;
- public SchedulerDriverModule(SchedulerMain.DriverKind kind) {
+ public SchedulerDriverModule(DriverKind kind) {
this.kind = kind;
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java
index bbccb17..ab98add 100644
--- a/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/offers/OffersModule.java
@@ -16,12 +16,13 @@ package org.apache.aurora.scheduler.offers;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
import java.util.List;
-import java.util.Set;
+
import javax.inject.Qualifier;
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Ordering;
import com.google.inject.AbstractModule;
import com.google.inject.Module;
@@ -29,14 +30,14 @@ import com.google.inject.PrivateModule;
import com.google.inject.Provides;
import com.google.inject.TypeLiteral;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.common.args.constraints.NotNegative;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.common.util.Random;
import org.apache.aurora.scheduler.HostOffer;
import org.apache.aurora.scheduler.app.MoreModules;
+import org.apache.aurora.scheduler.config.CliOptions;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
+import org.apache.aurora.scheduler.config.validators.NotNegativeAmount;
import org.apache.aurora.scheduler.events.PubsubEventModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,49 +53,56 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME;
public class OffersModule extends AbstractModule {
private static final Logger LOG = LoggerFactory.getLogger(OffersModule.class);
- @CmdLine(name = "min_offer_hold_time",
- help = "Minimum amount of time to hold a resource offer before declining.")
- @NotNegative
- private static final Arg<Amount<Integer, Time>> MIN_OFFER_HOLD_TIME =
- Arg.create(Amount.of(5, Time.MINUTES));
-
- @CmdLine(name = "offer_hold_jitter_window",
- help = "Maximum amount of random jitter to add to the offer hold time window.")
- @NotNegative
- private static final Arg<Amount<Integer, Time>> OFFER_HOLD_JITTER_WINDOW =
- Arg.create(Amount.of(1, Time.MINUTES));
-
- @CmdLine(name = "offer_filter_duration",
- help = "Duration after which we expect Mesos to re-offer unused resources. A short duration "
- + "improves scheduling performance in smaller clusters, but might lead to resource "
- + "starvation for other frameworks if you run many frameworks in your cluster.")
- private static final Arg<Amount<Long, Time>> OFFER_FILTER_DURATION =
- Arg.create(Amount.of(5L, Time.SECONDS));
-
- @CmdLine(name = "unavailability_threshold",
- help = "Threshold time, when running tasks should be drained from a host, before a host "
- + "becomes unavailable. Should be greater than min_offer_hold_time + "
- + "offer_hold_jitter_window.")
- private static final Arg<Amount<Long, Time>> UNAVAILABILITY_THRESHOLD =
- Arg.create(Amount.of(6L, Time.MINUTES));
-
- @CmdLine(name = "offer_order",
- help = "Iteration order for offers, to influence task scheduling. Multiple orderings will be "
- + "compounded together. E.g. CPU,MEMORY,RANDOM would sort first by cpus offered, then "
- + " memory and finally would randomize any equal offers.")
- private static final Arg<List<OfferOrder>> OFFER_ORDER =
- Arg.create(ImmutableList.of(OfferOrder.RANDOM));
-
- @CmdLine(name = "offer_order_modules",
- help = "Custom Guice module to provide an offer ordering.")
- private static final Arg<Set<Module>> OFFER_ORDER_MODULES = Arg.create(
- ImmutableSet.of(MoreModules.lazilyInstantiated(OfferOrderModule.class)));
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-min_offer_hold_time",
+ validateValueWith = NotNegativeAmount.class,
+ description = "Minimum amount of time to hold a resource offer before declining.")
+ public TimeAmount minOfferHoldTime = new TimeAmount(5, Time.MINUTES);
+
+ @Parameter(names = "-offer_hold_jitter_window",
+ validateValueWith = NotNegativeAmount.class,
+ description = "Maximum amount of random jitter to add to the offer hold time window.")
+ public TimeAmount offerHoldJitterWindow = new TimeAmount(1, Time.MINUTES);
+
+ @Parameter(names = "-offer_filter_duration",
+ description =
+ "Duration after which we expect Mesos to re-offer unused resources. A short duration "
+ + "improves scheduling performance in smaller clusters, but might lead to resource "
+ + "starvation for other frameworks if you run many frameworks in your cluster.")
+ public TimeAmount offerFilterDuration = new TimeAmount(5, Time.SECONDS);
+
+ @Parameter(names = "-unavailability_threshold",
+ description =
+ "Threshold time, when running tasks should be drained from a host, before a host "
+ + "becomes unavailable. Should be greater than min_offer_hold_time + "
+ + "offer_hold_jitter_window.")
+ public TimeAmount unavailabilityThreshold = new TimeAmount(6, Time.MINUTES);
+
+ @Parameter(names = "-offer_order",
+ description =
+ "Iteration order for offers, to influence task scheduling. Multiple orderings will be "
+ + "compounded together. E.g. CPU,MEMORY,RANDOM would sort first by cpus offered,"
+ + " then memory and finally would randomize any equal offers.")
+ public List<OfferOrder> offerOrder = ImmutableList.of(OfferOrder.RANDOM);
+
+ @Parameter(names = "-offer_order_modules",
+ description = "Custom Guice module to provide an offer ordering.")
+ @SuppressWarnings("rawtypes")
+ public List<Class> offerOrderModules = ImmutableList.of(OfferOrderModule.class);
+ }
public static class OfferOrderModule extends AbstractModule {
+ private final CliOptions options;
+
+ public OfferOrderModule(CliOptions options) {
+ this.options = options;
+ }
+
@Override
protected void configure() {
bind(new TypeLiteral<Ordering<HostOffer>>() { })
- .toInstance(OfferOrderBuilder.create(OFFER_ORDER.get()));
+ .toInstance(OfferOrderBuilder.create(options.offer.offerOrder));
}
}
@@ -105,26 +113,33 @@ public class OffersModule extends AbstractModule {
@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
public @interface UnavailabilityThreshold { }
+ private final CliOptions cliOptions;
+
+ public OffersModule(CliOptions cliOptions) {
+ this.cliOptions = cliOptions;
+ }
+
@Override
protected void configure() {
- long offerHoldTime = OFFER_HOLD_JITTER_WINDOW.get().as(Time.SECONDS)
- + MIN_OFFER_HOLD_TIME.get().as(Time.SECONDS);
- if (UNAVAILABILITY_THRESHOLD.get().as(Time.SECONDS) < offerHoldTime) {
+ Options options = cliOptions.offer;
+ long offerHoldTime =
+ options.offerHoldJitterWindow.as(Time.SECONDS) + options.minOfferHoldTime.as(Time.SECONDS);
+ if (options.unavailabilityThreshold.as(Time.SECONDS) < offerHoldTime) {
LOG.warn("unavailability_threshold ({}) is less than the sum of min_offer_hold_time ({}) and"
+ "offer_hold_jitter_window ({}). This creates risks of races between launching and"
+ "draining",
- UNAVAILABILITY_THRESHOLD.get(),
- MIN_OFFER_HOLD_TIME.get(),
- OFFER_HOLD_JITTER_WINDOW.get());
+ options.unavailabilityThreshold,
+ options.minOfferHoldTime,
+ options.offerHoldJitterWindow);
}
- for (Module module: OFFER_ORDER_MODULES.get()) {
+ for (Module module: MoreModules.instantiateAll(options.offerOrderModules, cliOptions)) {
install(module);
}
bind(new TypeLiteral<Amount<Long, Time>>() { })
.annotatedWith(UnavailabilityThreshold.class)
- .toInstance(UNAVAILABILITY_THRESHOLD.get());
+ .toInstance(options.unavailabilityThreshold);
install(new PrivateModule() {
@Override
@@ -141,10 +156,10 @@ public class OffersModule extends AbstractModule {
@Singleton
OfferSettings provideOfferSettings(Ordering<HostOffer> offerOrdering) {
return new OfferSettings(
- OFFER_FILTER_DURATION.get(),
+ cliOptions.offer.offerFilterDuration,
new RandomJitterReturnDelay(
- MIN_OFFER_HOLD_TIME.get().as(Time.MILLISECONDS),
- OFFER_HOLD_JITTER_WINDOW.get().as(Time.MILLISECONDS),
+ cliOptions.offer.minOfferHoldTime.as(Time.MILLISECONDS),
+ cliOptions.offer.offerHoldJitterWindow.as(Time.MILLISECONDS),
Random.Util.newDefaultRandom()),
offerOrdering);
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java b/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java
index 4b308dc..d3625ad 100644
--- a/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java
+++ b/src/main/java/org/apache/aurora/scheduler/offers/RandomJitterReturnDelay.java
@@ -30,11 +30,11 @@ import static com.google.common.base.Preconditions.checkArgument;
*/
@VisibleForTesting
class RandomJitterReturnDelay implements Supplier<Amount<Long, Time>> {
- private final int minHoldTimeMs;
- private final int maxJitterWindowMs;
+ private final long minHoldTimeMs;
+ private final long maxJitterWindowMs;
private final Random random;
- RandomJitterReturnDelay(int minHoldTimeMs, int maxJitterWindowMs, Random random) {
+ RandomJitterReturnDelay(long minHoldTimeMs, long maxJitterWindowMs, Random random) {
checkArgument(minHoldTimeMs >= 0);
checkArgument(maxJitterWindowMs >= 0);
@@ -45,6 +45,6 @@ class RandomJitterReturnDelay implements Supplier<Amount<Long, Time>> {
@Override
public Amount<Long, Time> get() {
- return Amount.of((long) minHoldTimeMs + random.nextInt(maxJitterWindowMs), Time.MILLISECONDS);
+ return Amount.of(minHoldTimeMs + random.nextInt((int) maxJitterWindowMs), Time.MILLISECONDS);
}
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java
index b3ca1a3..5d8907a 100644
--- a/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/preemptor/PreemptorModule.java
@@ -15,28 +15,30 @@ package org.apache.aurora.scheduler.preemptor;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
-import java.util.Set;
+import java.util.List;
+
import javax.inject.Inject;
import javax.inject.Qualifier;
import javax.inject.Singleton;
-import com.google.common.annotations.VisibleForTesting;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.AbstractScheduledService;
import com.google.inject.AbstractModule;
import com.google.inject.Module;
import com.google.inject.PrivateModule;
import com.google.inject.TypeLiteral;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.common.args.constraints.Positive;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.scheduler.SchedulerServicesModule;
import org.apache.aurora.scheduler.app.MoreModules;
import org.apache.aurora.scheduler.base.TaskGroupKey;
+import org.apache.aurora.scheduler.config.CliOptions;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
+import org.apache.aurora.scheduler.config.validators.PositiveNumber;
import org.apache.aurora.scheduler.events.PubsubEventModule;
import org.apache.aurora.scheduler.preemptor.BiCache.BiCacheSettings;
import org.slf4j.Logger;
@@ -52,42 +54,40 @@ public class PreemptorModule extends AbstractModule {
private static final Logger LOG = LoggerFactory.getLogger(PreemptorModule.class);
- @CmdLine(name = "enable_preemptor",
- help = "Enable the preemptor and preemption")
- private static final Arg<Boolean> ENABLE_PREEMPTOR = Arg.create(true);
-
- @CmdLine(name = "preemption_delay",
- help = "Time interval after which a pending task becomes eligible to preempt other tasks")
- private static final Arg<Amount<Long, Time>> PREEMPTION_DELAY =
- Arg.create(Amount.of(3L, Time.MINUTES));
-
- @CmdLine(name = "preemption_slot_hold_time",
- help = "Time to hold a preemption slot found before it is discarded.")
- private static final Arg<Amount<Long, Time>> PREEMPTION_SLOT_HOLD_TIME =
- Arg.create(Amount.of(5L, Time.MINUTES));
-
- @CmdLine(name = "preemption_slot_search_interval",
- help = "Time interval between pending task preemption slot searches.")
- private static final Arg<Amount<Long, Time>> PREEMPTION_SLOT_SEARCH_INTERVAL =
- Arg.create(Amount.of(1L, Time.MINUTES));
-
- @Positive
- @CmdLine(name = "preemption_reservation_max_batch_size",
- help = "The maximum number of reservations for a task group to be made in a batch.")
- private static final Arg<Integer> RESERVATION_MAX_BATCH_SIZE = Arg.create(5);
-
- @CmdLine(name = "preemption_slot_finder_modules",
- help = "Guice modules for custom preemption slot searching for pending tasks.")
- private static final Arg<Set<Module>> SLOT_FINDER_MODULES = Arg.create(
- ImmutableSet.of(
- MoreModules.lazilyInstantiated(PendingTaskProcessorModule.class),
- MoreModules.lazilyInstantiated(PreemptionVictimFilterModule.class)));
-
- private final boolean enablePreemptor;
- private final Amount<Long, Time> preemptionDelay;
- private final Amount<Long, Time> slotSearchInterval;
- private final Integer reservationBatchSize;
- private final Set<Module> slotFinderModules;
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-enable_preemptor",
+ description = "Enable the preemptor and preemption",
+ arity = 1)
+ public boolean enablePreemptor = true;
+
+ @Parameter(names = "-preemption_delay",
+ description =
+ "Time interval after which a pending task becomes eligible to preempt other tasks")
+ public TimeAmount preemptionDelay = new TimeAmount(3, Time.MINUTES);
+
+ @Parameter(names = "-preemption_slot_hold_time",
+ description = "Time to hold a preemption slot found before it is discarded.")
+ public TimeAmount preemptionSlotHoldTime = new TimeAmount(5, Time.MINUTES);
+
+ @Parameter(names = "-preemption_slot_search_interval",
+ description = "Time interval between pending task preemption slot searches.")
+ public TimeAmount preemptionSlotSearchInterval = new TimeAmount(1, Time.MINUTES);
+
+ @Parameter(names = "-preemption_reservation_max_batch_size",
+ validateValueWith = PositiveNumber.class,
+ description = "The maximum number of reservations for a task group to be made in a batch.")
+ public int reservationMaxBatchSize = 5;
+
+ @Parameter(names = "-preemption_slot_finder_modules",
+ description = "Guice modules for custom preemption slot searching for pending tasks.")
+ @SuppressWarnings("rawtypes")
+ public List<Class> slotFinderModules = ImmutableList.of(
+ PendingTaskProcessorModule.class,
+ PreemptionVictimFilterModule.class);
+ }
+
+ private final CliOptions cliOptions;
/*
* Binding annotation for the async processor that finds preemption slots.
@@ -96,70 +96,36 @@ public class PreemptorModule extends AbstractModule {
@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
public @interface PreemptionSlotFinder { }
- @VisibleForTesting
- public PreemptorModule(
- boolean enablePreemptor,
- Amount<Long, Time> preemptionDelay,
- Amount<Long, Time> slotSearchInterval,
- Integer reservationBatchSize,
- Set<Module> slotFinderModules) {
-
- this.enablePreemptor = enablePreemptor;
- this.preemptionDelay = requireNonNull(preemptionDelay);
- this.slotSearchInterval = requireNonNull(slotSearchInterval);
- this.reservationBatchSize = requireNonNull(reservationBatchSize);
- this.slotFinderModules = requireNonNull(slotFinderModules);
- }
-
- @VisibleForTesting
- public PreemptorModule(
- boolean enablePreemptor,
- Amount<Long, Time> preemptionDelay,
- Amount<Long, Time> slotSearchInterval,
- Integer reservationBatchSize) {
-
- this(
- enablePreemptor,
- preemptionDelay,
- slotSearchInterval,
- reservationBatchSize,
- SLOT_FINDER_MODULES.get());
- }
-
- public PreemptorModule() {
- this(
- ENABLE_PREEMPTOR.get(),
- PREEMPTION_DELAY.get(),
- PREEMPTION_SLOT_SEARCH_INTERVAL.get(),
- RESERVATION_MAX_BATCH_SIZE.get(),
- SLOT_FINDER_MODULES.get());
+ public PreemptorModule(CliOptions cliOptions) {
+ this.cliOptions = cliOptions;
}
@Override
protected void configure() {
+ Options options = cliOptions.preemptor;
install(new PrivateModule() {
@Override
protected void configure() {
- if (enablePreemptor) {
+ if (options.enablePreemptor) {
LOG.info("Preemptor Enabled.");
bind(PreemptorMetrics.class).in(Singleton.class);
bind(Preemptor.class).to(Preemptor.PreemptorImpl.class);
bind(Preemptor.PreemptorImpl.class).in(Singleton.class);
bind(new TypeLiteral<Amount<Long, Time>>() { })
.annotatedWith(PendingTaskProcessor.PreemptionDelay.class)
- .toInstance(preemptionDelay);
+ .toInstance(options.preemptionDelay);
bind(BiCacheSettings.class).toInstance(
- new BiCacheSettings(PREEMPTION_SLOT_HOLD_TIME.get(), "preemption_slot"));
+ new BiCacheSettings(options.preemptionSlotHoldTime, "preemption_slot"));
bind(new TypeLiteral<BiCache<PreemptionProposal, TaskGroupKey>>() { })
.in(Singleton.class);
bind(new TypeLiteral<Integer>() { })
.annotatedWith(PendingTaskProcessor.ReservationBatchSize.class)
- .toInstance(reservationBatchSize);
+ .toInstance(options.reservationMaxBatchSize);
bind(ClusterState.class).to(ClusterStateImpl.class);
bind(ClusterStateImpl.class).in(Singleton.class);
expose(ClusterStateImpl.class);
- for (Module module: slotFinderModules) {
+ for (Module module: MoreModules.instantiateAll(options.slotFinderModules, cliOptions)) {
install(module);
}
@@ -167,8 +133,8 @@ public class PreemptorModule extends AbstractModule {
bind(AbstractScheduledService.Scheduler.class).toInstance(
AbstractScheduledService.Scheduler.newFixedRateSchedule(
0L,
- slotSearchInterval.getValue(),
- slotSearchInterval.getUnit().getTimeUnit()));
+ options.preemptionSlotSearchInterval.getValue(),
+ options.preemptionSlotSearchInterval.getUnit().getTimeUnit()));
expose(PreemptorService.class);
expose(Runnable.class).annotatedWith(PreemptionSlotFinder.class);
@@ -184,7 +150,7 @@ public class PreemptorModule extends AbstractModule {
// and private modules due to multiple injectors. We accept the added complexity here to keep
// the other bindings private.
PubsubEventModule.bindSubscriber(binder(), ClusterStateImpl.class);
- if (enablePreemptor) {
+ if (options.enablePreemptor) {
SchedulerServicesModule.addSchedulerActiveServiceBinding(binder())
.to(PreemptorService.class);
}
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java b/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java
index 735199a..4433b96 100644
--- a/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/pruning/PruningModule.java
@@ -17,15 +17,15 @@ import java.util.concurrent.ScheduledExecutorService;
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.inject.AbstractModule;
import com.google.inject.PrivateModule;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.scheduler.SchedulerServicesModule;
import org.apache.aurora.scheduler.base.AsyncUtil;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
import org.apache.aurora.scheduler.events.PubsubEventModule;
import org.apache.aurora.scheduler.pruning.TaskHistoryPruner.HistoryPrunnerSettings;
import org.slf4j.Logger;
@@ -38,33 +38,39 @@ public class PruningModule extends AbstractModule {
private static final Logger LOG = LoggerFactory.getLogger(PruningModule.class);
- @CmdLine(name = "history_prune_threshold",
- help = "Time after which the scheduler will prune terminated task history.")
- private static final Arg<Amount<Long, Time>> HISTORY_PRUNE_THRESHOLD =
- Arg.create(Amount.of(2L, Time.DAYS));
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-history_prune_threshold",
+ description = "Time after which the scheduler will prune terminated task history.")
+ public TimeAmount historyPruneThreshold = new TimeAmount(2, Time.DAYS);
- @CmdLine(name = "history_max_per_job_threshold",
- help = "Maximum number of terminated tasks to retain in a job history.")
- private static final Arg<Integer> HISTORY_MAX_PER_JOB_THRESHOLD = Arg.create(100);
+ @Parameter(names = "-history_max_per_job_threshold",
+ description = "Maximum number of terminated tasks to retain in a job history.")
+ public int historyMaxPerJobThreshold = 100;
- @CmdLine(name = "history_min_retention_threshold",
- help = "Minimum guaranteed time for task history retention before any pruning is attempted.")
- private static final Arg<Amount<Long, Time>> HISTORY_MIN_RETENTION_THRESHOLD =
- Arg.create(Amount.of(1L, Time.HOURS));
+ @Parameter(names = "-history_min_retention_threshold",
+ description =
+ "Minimum guaranteed time for task history retention before any pruning is attempted.")
+ public TimeAmount historyMinRetentionThreshold = new TimeAmount(1, Time.HOURS);
- @CmdLine(name = "job_update_history_per_job_threshold",
- help = "Maximum number of completed job updates to retain in a job update history.")
- private static final Arg<Integer> JOB_UPDATE_HISTORY_PER_JOB_THRESHOLD = Arg.create(10);
+ @Parameter(names = "-job_update_history_per_job_threshold",
+ description = "Maximum number of completed job updates to retain in a job update history.")
+ public int jobUpdateHistoryPerJobThreshold = 10;
- @CmdLine(name = "job_update_history_pruning_interval",
- help = "Job update history pruning interval.")
- private static final Arg<Amount<Long, Time>> JOB_UPDATE_HISTORY_PRUNING_INTERVAL =
- Arg.create(Amount.of(15L, Time.MINUTES));
+ @Parameter(names = "-job_update_history_pruning_interval",
+ description = "Job update history pruning interval.")
+ public TimeAmount jobUpdateHistoryPruningInterval = new TimeAmount(15, Time.MINUTES);
- @CmdLine(name = "job_update_history_pruning_threshold",
- help = "Time after which the scheduler will prune completed job update history.")
- private static final Arg<Amount<Long, Time>> JOB_UPDATE_HISTORY_PRUNING_THRESHOLD =
- Arg.create(Amount.of(30L, Time.DAYS));
+ @Parameter(names = "-job_update_history_pruning_threshold",
+ description = "Time after which the scheduler will prune completed job update history.")
+ public TimeAmount jobUpdateHistoryPruningThreshold = new TimeAmount(30, Time.DAYS);
+ }
+
+ private final Options options;
+
+ public PruningModule(Options options) {
+ this.options = options;
+ }
@Override
protected void configure() {
@@ -74,9 +80,9 @@ public class PruningModule extends AbstractModule {
// TODO(ksweeney): Create a configuration validator module so this can be injected.
// TODO(William Farner): Revert this once large task counts is cheap ala hierarchichal store
bind(HistoryPrunnerSettings.class).toInstance(new HistoryPrunnerSettings(
- HISTORY_PRUNE_THRESHOLD.get(),
- HISTORY_MIN_RETENTION_THRESHOLD.get(),
- HISTORY_MAX_PER_JOB_THRESHOLD.get()
+ options.historyPruneThreshold,
+ options.historyMinRetentionThreshold,
+ options.historyMaxPerJobThreshold
));
bind(TaskHistoryPruner.class).in(Singleton.class);
@@ -90,9 +96,9 @@ public class PruningModule extends AbstractModule {
protected void configure() {
bind(JobUpdateHistoryPruner.HistoryPrunerSettings.class).toInstance(
new JobUpdateHistoryPruner.HistoryPrunerSettings(
- JOB_UPDATE_HISTORY_PRUNING_INTERVAL.get(),
- JOB_UPDATE_HISTORY_PRUNING_THRESHOLD.get(),
- JOB_UPDATE_HISTORY_PER_JOB_THRESHOLD.get()));
+ options.jobUpdateHistoryPruningInterval,
+ options.jobUpdateHistoryPruningThreshold,
+ options.jobUpdateHistoryPerJobThreshold));
bind(ScheduledExecutorService.class).toInstance(
AsyncUtil.singleThreadLoggingScheduledExecutor("JobUpdatePruner-%d", LOG));
http://git-wip-us.apache.org/repos/asf/aurora/blob/519e3df7/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java b/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java
index 80fc616..7af00e5 100644
--- a/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/reconciliation/ReconciliationModule.java
@@ -20,19 +20,21 @@ import java.util.concurrent.ScheduledExecutorService;
import javax.inject.Qualifier;
import javax.inject.Singleton;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
import com.google.inject.AbstractModule;
import com.google.inject.PrivateModule;
import com.google.inject.TypeLiteral;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.common.args.constraints.Positive;
import org.apache.aurora.common.quantity.Amount;
import org.apache.aurora.common.quantity.Time;
import org.apache.aurora.common.util.BackoffStrategy;
import org.apache.aurora.common.util.TruncatedBinaryBackoff;
import org.apache.aurora.scheduler.SchedulerServicesModule;
import org.apache.aurora.scheduler.base.AsyncUtil;
+import org.apache.aurora.scheduler.config.types.TimeAmount;
+import org.apache.aurora.scheduler.config.validators.PositiveAmount;
+import org.apache.aurora.scheduler.config.validators.PositiveNumber;
import org.apache.aurora.scheduler.events.PubsubEventModule;
import org.apache.aurora.scheduler.reconciliation.TaskReconciler.TaskReconcilerSettings;
import org.slf4j.Logger;
@@ -50,66 +52,72 @@ public class ReconciliationModule extends AbstractModule {
private static final Logger LOG = LoggerFactory.getLogger(ReconciliationModule.class);
- @CmdLine(name = "transient_task_state_timeout",
- help = "The amount of time after which to treat a task stuck in a transient state as LOST.")
- private static final Arg<Amount<Long, Time>> TRANSIENT_TASK_STATE_TIMEOUT =
- Arg.create(Amount.of(5L, Time.MINUTES));
-
- @CmdLine(name = "initial_task_kill_retry_interval",
- help = "When killing a task, retry after this delay if mesos has not responded,"
- + " backing off up to transient_task_state_timeout")
- private static final Arg<Amount<Long, Time>> INITIAL_TASK_KILL_RETRY_INTERVAL =
- Arg.create(Amount.of(15L, Time.SECONDS));
-
- // Reconciliation may create a big surge of status updates in a large cluster. Setting the default
- // initial delay to 1 minute to ease up storage contention during scheduler start up.
- @CmdLine(name = "reconciliation_initial_delay",
- help = "Initial amount of time to delay task reconciliation after scheduler start up.")
- private static final Arg<Amount<Long, Time>> RECONCILIATION_INITIAL_DELAY =
- Arg.create(Amount.of(1L, Time.MINUTES));
-
- @Positive
- @CmdLine(name = "reconciliation_explicit_interval",
- help = "Interval on which scheduler will ask Mesos for status updates of all non-terminal "
- + "tasks known to scheduler.")
- private static final Arg<Amount<Long, Time>> RECONCILIATION_EXPLICIT_INTERVAL =
- Arg.create(Amount.of(60L, Time.MINUTES));
-
- @Positive
- @CmdLine(name = "reconciliation_implicit_interval",
- help = "Interval on which scheduler will ask Mesos for status updates of all non-terminal "
- + "tasks known to Mesos.")
- private static final Arg<Amount<Long, Time>> RECONCILIATION_IMPLICIT_INTERVAL =
- Arg.create(Amount.of(60L, Time.MINUTES));
-
- @CmdLine(name = "reconciliation_schedule_spread",
- help = "Difference between explicit and implicit reconciliation intervals intended to "
- + "create a non-overlapping task reconciliation schedule.")
- private static final Arg<Amount<Long, Time>> RECONCILIATION_SCHEDULE_SPREAD =
- Arg.create(Amount.of(30L, Time.MINUTES));
-
- @Positive
- @CmdLine(name = "reconciliation_explicit_batch_size",
- help = "Number of tasks in a single batch request sent to Mesos for explicit reconciliation.")
- private static final Arg<Integer> RECONCILIATION_BATCH_SIZE = Arg.create(1000);
-
- @Positive
- @CmdLine(name = "reconciliation_explicit_batch_interval",
- help = "Interval between explicit batch reconciliation requests.")
- private static final Arg<Amount<Long, Time>> RECONCILIATION_BATCH_INTERVAL =
- Arg.create(Amount.of(5L, Time.SECONDS));
+ @Parameters(separators = "=")
+ public static class Options {
+ @Parameter(names = "-transient_task_state_timeout",
+ description =
+ "The amount of time after which to treat a task stuck in a transient state as LOST.")
+ public TimeAmount transientTaskStateTimeout = new TimeAmount(5L, Time.MINUTES);
+
+ @Parameter(names = "-initial_task_kill_retry_interval",
+ description = "When killing a task, retry after this delay if mesos has not responded,"
+ + " backing off up to transient_task_state_timeout")
+ public TimeAmount initialTaskKillRetryInterval = new TimeAmount(15L, Time.SECONDS);
+
+ // Reconciliation may create a big surge of status updates in a large cluster. Setting the
+ // default initial delay to 1 minute to ease up storage contention during scheduler start up.
+ @Parameter(names = "-reconciliation_initial_delay",
+ description =
+ "Initial amount of time to delay task reconciliation after scheduler start up.")
+ public TimeAmount reconciliationInitialDelay = new TimeAmount(1L, Time.MINUTES);
+
+ @Parameter(names = "-reconciliation_explicit_interval",
+ validateValueWith = PositiveAmount.class,
+ description = "Interval on which scheduler will ask Mesos for status updates of all"
+ + "non-terminal tasks known to scheduler.")
+ public TimeAmount reconciliationExplicitInterval = new TimeAmount(60L, Time.MINUTES);
+
+ @Parameter(names = "-reconciliation_implicit_interval",
+ validateValueWith = PositiveAmount.class,
+ description = "Interval on which scheduler will ask Mesos for status updates of all"
+ + "non-terminal tasks known to Mesos.")
+ public TimeAmount reconciliationImplicitInterval = new TimeAmount(60L, Time.MINUTES);
+
+ @Parameter(names = "-reconciliation_schedule_spread",
+ description =
+ "Difference between explicit and implicit reconciliation intervals intended to "
+ + "create a non-overlapping task reconciliation schedule.")
+ public TimeAmount reconciliationScheduleSpread = new TimeAmount(30L, Time.MINUTES);
+
+ @Parameter(names = "-reconciliation_explicit_batch_size",
+ validateValueWith = PositiveNumber.class,
+ description =
+ "Number of tasks in a single batch request sent to Mesos for explicit reconciliation.")
+ public int reconciliationBatchSize = 1000;
+
+ @Parameter(names = "-reconciliation_explicit_batch_interval",
+ validateValueWith = PositiveAmount.class,
+ description = "Interval between explicit batch reconciliation requests.")
+ public TimeAmount reconciliationBatchInterval = new TimeAmount(5L, Time.SECONDS);
+ }
@Qualifier
@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
@interface BackgroundWorker { }
+ private final Options options;
+
+ public ReconciliationModule(Options options) {
+ this.options = options;
+ }
+
@Override
protected void configure() {
install(new PrivateModule() {
@Override
protected void configure() {
bind(new TypeLiteral<Amount<Long, Time>>() { })
- .toInstance(TRANSIENT_TASK_STATE_TIMEOUT.get());
+ .toInstance(options.transientTaskStateTimeout);
bind(TaskTimeout.class).in(Singleton.class);
expose(TaskTimeout.class);
@@ -123,8 +131,8 @@ public class ReconciliationModule extends AbstractModule {
protected void configure() {
bind(BackoffStrategy.class).toInstance(
new TruncatedBinaryBackoff(
- INITIAL_TASK_KILL_RETRY_INTERVAL.get(),
- TRANSIENT_TASK_STATE_TIMEOUT.get()));
+ options.initialTaskKillRetryInterval,
+ options.transientTaskStateTimeout));
bind(KillRetry.class).in(Singleton.class);
expose(KillRetry.class);
}
@@ -135,12 +143,12 @@ public class ReconciliationModule extends AbstractModule {
@Override
protected void configure() {
bind(TaskReconcilerSettings.class).toInstance(new TaskReconcilerSettings(
- RECONCILIATION_INITIAL_DELAY.get(),
- RECONCILIATION_EXPLICIT_INTERVAL.get(),
- RECONCILIATION_IMPLICIT_INTERVAL.get(),
- RECONCILIATION_SCHEDULE_SPREAD.get(),
- RECONCILIATION_BATCH_INTERVAL.get(),
- RECONCILIATION_BATCH_SIZE.get()));
+ options.reconciliationInitialDelay,
+ options.reconciliationExplicitInterval,
+ options.reconciliationImplicitInterval,
+ options.reconciliationScheduleSpread,
+ options.reconciliationBatchInterval,
+ options.reconciliationBatchSize));
bind(ScheduledExecutorService.class).annotatedWith(BackgroundWorker.class)
.toInstance(AsyncUtil.loggingScheduledExecutor(1, "TaskReconciler-%d", LOG));
bind(TaskReconciler.class).in(Singleton.class);