You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/11/11 11:01:44 UTC
[01/50] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
[Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5881 bade66cb0 -> 6df85aaa2 (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index f22d487..df4b1df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.*;
public class TestServiceCLI {
+ private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
+ .class);
- protected Configuration conf = new YarnConfiguration();
+ private Configuration conf = new YarnConfiguration();
private File basedir;
- private ServiceCLI cli;
private SliderFileSystem fs;
+ private String basedirProp;
- private void buildApp(String appDef) throws Throwable {
- String[] args =
- { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- cli.exec(clientArgs);
+ private void runCLI(String[] args) throws Exception {
+ LOG.info("running CLI: yarn {}", Arrays.asList(args));
+ ApplicationCLI cli = new ApplicationCLI();
+ cli.setSysOutPrintStream(System.out);
+ cli.setSysErrPrintStream(System.err);
+ int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+ cli.stop();
+ }
+
+ private void buildApp(String serviceName, String appDef) throws Throwable {
+ String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef)};
+ runCLI(args);
+ }
+
+ private void buildApp(String serviceName, String appDef, String lifetime,
+ String queue) throws Throwable {
+ String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
+ "-changeQueue", queue};
+ runCLI(args);
}
@Before
public void setup() throws Throwable {
basedir = new File("target", "apps");
+ basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
- conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
- conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
- conf.setInt(
- CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- conf.setInt(CommonConfigurationKeysPublic.
- IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
fs = new SliderFileSystem(conf);
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
} else {
basedir.mkdirs();
}
-
- // create a CLI and skip connection to AM
- cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient() {
- @Override
- protected void serviceInit(Configuration configuration)
- throws Exception {
- super.serviceInit(conf);
- yarnClient = spy(yarnClient);
- ApplicationReport report = Records.newRecord(ApplicationReport.class);
- report.setYarnApplicationState(YarnApplicationState.RUNNING);
- report.setHost("localhost");
- doReturn(report).when(yarnClient).getApplicationReport(anyObject());
- }
- @Override
- protected ClientAMProtocol createAMProxy(String host, int port)
- throws IOException {
- return mock(ClientAMProtocol.class);
- }
- };
- client.init(conf);
- client.start();
- }
- };
}
@After
@@ -111,41 +94,31 @@ public class TestServiceCLI {
}
}
- // Test flex components count are persisted.
@Test
public void testFlexComponents() throws Throwable {
+ // currently can only test building apps, since that is the only
+ // operation that doesn't require an RM
+ // TODO: expand CLI test to try other commands
String serviceName = "app-1";
- buildApp(ExampleAppJson.APP_JSON);
- checkCompCount("master",serviceName, 1L);
-
- // increase by 2
- String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"};
- ClientArgs clientArgs = new ClientArgs(flexUpArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 3L);
-
- // decrease by 1
- String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"};
- clientArgs = new ClientArgs(flexDownArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 2L);
+ buildApp(serviceName, ExampleAppJson.APP_JSON);
+ checkApp(serviceName, "master", 1L, 3600L, null);
- String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"};
- clientArgs = new ClientArgs(flexAbsoluteArgs);
- clientArgs.parse();
- cli.exec(clientArgs);
- checkCompCount("master", serviceName, 10L);
+ serviceName = "app-2";
+ buildApp(serviceName, ExampleAppJson.APP_JSON, "1000", "qname");
+ checkApp(serviceName, "master", 1L, 1000L, "qname");
}
- private void checkCompCount(String compName, String serviceName, long count)
- throws IOException {
- List<Component> components =
- ServiceApiUtil.getComponents(fs, serviceName);
+ private void checkApp(String serviceName, String compName, long count, Long
+ lifetime, String queue) throws IOException {
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ Assert.assertEquals(serviceName, service.getName());
+ Assert.assertEquals(lifetime, service.getLifetime());
+ Assert.assertEquals(queue, service.getQueue());
+ List<Component> components = service.getComponents();
for (Component component : components) {
if (component.getName().equals(compName)) {
- Assert.assertEquals(count, component.getNumberOfContainers().longValue());
+ Assert.assertEquals(count, component.getNumberOfContainers()
+ .longValue());
return;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
new file mode 100644
index 0000000..6310178
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Client for managing applications.
+ */
+@Public
+@Unstable
+public abstract class AppAdminClient extends CompositeService {
+ public static final String YARN_APP_ADMIN_CLIENT_PREFIX = "yarn" +
+ ".application.admin.client.class.";
+ public static final String DEFAULT_TYPE = "yarn-service";
+ public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" +
+ ".service.client.ServiceClient";
+
+ @Private
+ protected AppAdminClient() {
+ super(AppAdminClient.class.getName());
+ }
+
+ /**
+ * <p>
+ * Create a new instance of AppAdminClient.
+ * </p>
+ *
+ * @param appType application type
+ * @param conf configuration
+ * @return app admin client
+ */
+ @Public
+ @Unstable
+ public static AppAdminClient createAppAdminClient(String appType,
+ Configuration conf) {
+ Map<String, String> clientClassMap =
+ conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX);
+ if (!clientClassMap.containsKey(DEFAULT_TYPE)) {
+ clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME);
+ }
+ if (!clientClassMap.containsKey(appType)) {
+ throw new IllegalArgumentException("App admin client class name not " +
+ "specified for type " + appType);
+ }
+ String clientClassName = clientClassMap.get(appType);
+ Class<? extends AppAdminClient> clientClass;
+ try {
+ clientClass = (Class<? extends AppAdminClient>) Class.forName(
+ clientClassName);
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Invalid app admin client class", e);
+ }
+
+ AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClass,
+ conf);
+ appAdminClient.init(conf);
+ appAdminClient.start();
+ return appAdminClient;
+ }
+
+ /**
+ * <p>
+ * Launch a new YARN application.
+ * </p>
+ *
+ * @param fileName specification of application
+ * @param appName name of the application
+ * @param lifetime lifetime of the application
+ * @param queue queue of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionLaunch(String fileName, String appName, Long
+ lifetime, String queue) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Stop a YARN application (attempt to stop gracefully before killing the
+ * application). In the case of a long-running service, the service may be
+ * restarted later.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionStop(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Start a YARN application from a previously saved specification. In the
+ * case of a long-running service, the service must have been previously
+ * launched/started and then stopped, or previously saved but not started.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionStart(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Save the specification for a YARN application / long-running service.
+ * The application may be started later.
+ * </p>
+ *
+ * @param fileName specification of application to save
+ * @param appName name of the application
+ * @param lifetime lifetime of the application
+ * @param queue queue of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionSave(String fileName, String appName, Long
+ lifetime, String queue) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Remove the specification and all application data for a YARN application.
+ * The application cannot be running.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionDestroy(String appName) throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Change the number of running containers for a component of a YARN
+ * application / long-running service.
+ * </p>
+ *
+ * @param appName the name of the application
+ * @param componentCounts map of component name to new component count or
+ * amount to change existing component count (e.g.
+ * 5, +5, -5)
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionFlex(String appName, Map<String, String>
+ componentCounts) throws IOException, YarnException;
+
+ /**
+ * <p>
+ * Upload AM dependencies to HDFS. This makes future application launches
+ * faster since the dependencies do not have to be uploaded on each launch.
+ * </p>
+ *
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int enableFastLaunch() throws IOException,
+ YarnException;
+
+ /**
+ * <p>
+ * Get detailed status string for a YARN application.
+ * </p>
+ *
+ * @param applicationId application id
+ * @return status string
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract String getStatusString(String applicationId) throws
+ IOException, YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 2a9b3bc..fb08fcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -23,12 +23,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -54,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -85,6 +81,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
+ public static final String APP = "app";
public static final String APPLICATION = "application";
public static final String APPLICATION_ATTEMPT = "applicationattempt";
public static final String CONTAINER = "container";
@@ -93,22 +90,52 @@ public class ApplicationCLI extends YarnCLI {
public static final String UPDATE_LIFETIME = "updateLifetime";
public static final String CHANGE_APPLICATION_QUEUE = "changeQueue";
+ // app admin options
+ public static final String LAUNCH_CMD = "launch";
+ public static final String STOP_CMD = "stop";
+ public static final String START_CMD = "start";
+ public static final String SAVE_CMD = "save";
+ public static final String DESTROY_CMD = "destroy";
+ public static final String FLEX_CMD = "flex";
+ public static final String COMPONENT = "component";
+ public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch";
+
+ private static String firstArg = null;
+
private boolean allAppStates;
public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
- int res = ToolRunner.run(cli, args);
+ int res = ToolRunner.run(cli, preProcessArgs(args));
cli.stop();
System.exit(res);
}
+ @VisibleForTesting
+ public static String[] preProcessArgs(String[] args) {
+ if (args.length > 0) {
+ // first argument (app|application|applicationattempt|container) must
+ // be stripped off for GenericOptionsParser to work
+ firstArg = args[0];
+ return Arrays.copyOfRange(args, 1, args.length);
+ } else {
+ return args;
+ }
+ }
+
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
String title = null;
- if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
+ if (firstArg != null) {
+ title = firstArg;
+ } else if (args.length > 0) {
+ title = args[0];
+ }
+ if (title != null && (title.equalsIgnoreCase(APPLICATION) || title
+ .equalsIgnoreCase(APP))) {
title = APPLICATION;
opts.addOption(STATUS_CMD, true,
"Prints the status of the application.");
@@ -168,8 +195,52 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
- } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
- title = APPLICATION_ATTEMPT;
+ opts.addOption(LAUNCH_CMD, true, "Launches application from " +
+ "specification file (saves specification and starts application). " +
+ "Options -updateLifetime and -changeQueue can be specified to alter" +
+ " the values provided in the file. Supports -appTypes option to " +
+ "specify which client implementation to use.");
+ opts.addOption(STOP_CMD, true, "Stops application gracefully (may be " +
+ "started again later). If name is provided, appType must be " +
+ "provided unless it is the default yarn-service. If ID is provided," +
+ " the appType will be looked up. Supports -appTypes option to " +
+ "specify which client implementation to use.");
+ opts.addOption(START_CMD, true, "Starts a previously saved " +
+ "application. Supports -appTypes option to specify which client " +
+ "implementation to use.");
+ opts.addOption(SAVE_CMD, true, "Saves specification file for " +
+ "an application. Options -updateLifetime and -changeQueue can be " +
+ "specified to alter the values provided in the file. Supports " +
+ "-appTypes option to specify which client implementation to use.");
+ opts.addOption(DESTROY_CMD, true, "Destroys a saved application " +
+ "specification and removes all application data permanently. " +
+ "Supports -appTypes option to specify which client implementation " +
+ "to use.");
+ opts.addOption(FLEX_CMD, true, "Changes number of " +
+ "running containers for a component of an application / " +
+ "long-running service. Requires -component option. If name is " +
+ "provided, appType must be provided unless it is the default " +
+ "yarn-service. If ID is provided, the appType will be looked up. " +
+ "Supports -appTypes option to specify which client implementation " +
+ "to use.");
+ opts.addOption(COMPONENT, true, "Works with -flex option to change " +
+ "the number of components/containers running for an application / " +
+ "long-running service. Supports absolute or relative changes, such " +
+ "as +1, 2, or -3.");
+ opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " +
+ "to HDFS to make future launches faster. Supports -appTypes option" +
+ " to specify which client implementation to use.");
+ opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name");
+ opts.getOption(LAUNCH_CMD).setArgs(2);
+ opts.getOption(START_CMD).setArgName("Application Name");
+ opts.getOption(STOP_CMD).setArgName("Application Name or ID");
+ opts.getOption(SAVE_CMD).setArgName("Application Name> <File Name");
+ opts.getOption(SAVE_CMD).setArgs(2);
+ opts.getOption(DESTROY_CMD).setArgName("Application Name");
+ opts.getOption(FLEX_CMD).setArgName("Application Name or ID");
+ opts.getOption(COMPONENT).setArgName("Component Name> <Count");
+ opts.getOption(COMPONENT).setArgs(2);
+ } else if (title != null && title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the application attempt.");
opts.addOption(LIST_CMD, true,
@@ -179,8 +250,7 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
opts.getOption(LIST_CMD).setArgName("Application ID");
opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
- } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
- title = CONTAINER;
+ } else if (title != null && title.equalsIgnoreCase(CONTAINER)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the container.");
opts.addOption(LIST_CMD, true,
@@ -205,23 +275,53 @@ public class ApplicationCLI extends YarnCLI {
printUsage(title, opts);
return exitCode;
}
+ String[] unparsedArgs = cliParser.getArgs();
+ if (firstArg == null) {
+ if (unparsedArgs.length != 1) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ } else {
+ if (unparsedArgs.length != 0) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ }
if (cliParser.hasOption(STATUS_CMD)) {
- if (args.length != 3) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, STATUS_CMD)) {
printUsage(title, opts);
return exitCode;
}
- if (args[0].equalsIgnoreCase(APPLICATION)) {
- exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
- } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (title.equalsIgnoreCase(APPLICATION) ||
+ title.equalsIgnoreCase(APP)) {
+ ApplicationReport report = printApplicationReport(cliParser
+ .getOptionValue(STATUS_CMD));
+ if (report == null) {
+ exitCode = -1;
+ } else {
+ exitCode = 0;
+ String appType = report.getApplicationType();
+ try {
+ AppAdminClient client = AppAdminClient.createAppAdminClient(appType,
+ getConf());
+ sysout.println("Detailed Application Status :");
+ sysout.println(client.getStatusString(cliParser.getOptionValue(
+ STATUS_CMD)));
+ } catch (IllegalArgumentException e) {
+ // app type does not have app admin client implementation
+ }
+ }
+ } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
exitCode = printApplicationAttemptReport(cliParser
.getOptionValue(STATUS_CMD));
- } else if (args[0].equalsIgnoreCase(CONTAINER)) {
+ } else if (title.equalsIgnoreCase(CONTAINER)) {
exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
}
return exitCode;
} else if (cliParser.hasOption(LIST_CMD)) {
- if (args[0].equalsIgnoreCase(APPLICATION)) {
+ if (title.equalsIgnoreCase(APPLICATION) ||
+ title.equalsIgnoreCase(APP)) {
allAppStates = false;
Set<String> appTypes = new HashSet<String>();
if (cliParser.hasOption(APP_TYPE_CMD)) {
@@ -272,21 +372,21 @@ public class ApplicationCLI extends YarnCLI {
}
}
listApplications(appTypes, appStates, appTags);
- } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
- if (args.length != 3) {
+ } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
- } else if (args[0].equalsIgnoreCase(CONTAINER)) {
- if (args.length != 3) {
+ } else if (title.equalsIgnoreCase(CONTAINER)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listContainers(cliParser.getOptionValue(LIST_CMD));
}
} else if (cliParser.hasOption(KILL_CMD)) {
- if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
printUsage(title, opts);
return exitCode;
}
@@ -299,7 +399,7 @@ public class ApplicationCLI extends YarnCLI {
moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD),
cliParser.getOptionValue(QUEUE_CMD));
} else if (cliParser.hasOption(FAIL_CMD)) {
- if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
+ if (!title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
printUsage(title, opts);
return exitCode;
}
@@ -314,6 +414,103 @@ public class ApplicationCLI extends YarnCLI {
}
updateApplicationPriority(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(UPDATE_PRIORITY));
+ } else if (cliParser.hasOption(SIGNAL_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, SIGNAL_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
+ final String containerId = signalArgs[0];
+ SignalContainerCommand command =
+ SignalContainerCommand.OUTPUT_THREAD_DUMP;
+ if (signalArgs.length == 2) {
+ command = SignalContainerCommand.valueOf(signalArgs[1]);
+ }
+ signalToContainer(containerId, command);
+ } else if (cliParser.hasOption(LAUNCH_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD,
+ UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ Long lifetime = null;
+ if (cliParser.hasOption(UPDATE_LIFETIME)) {
+ lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
+ }
+ String queue = null;
+ if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
+ queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
+ }
+ String[] nameAndFile = cliParser.getOptionValues(LAUNCH_CMD);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionLaunch(nameAndFile[1], nameAndFile[0], lifetime, queue);
+ } else if (cliParser.hasOption(STOP_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, STOP_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String[] appNameAndType = getAppNameAndType(cliParser, STOP_CMD);
+ return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
+ .actionStop(appNameAndType[0]);
+ } else if (cliParser.hasOption(START_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, START_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionStart(cliParser.getOptionValue(START_CMD));
+ } else if (cliParser.hasOption(SAVE_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, SAVE_CMD, APP_TYPE_CMD,
+ UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ Long lifetime = null;
+ if (cliParser.hasOption(UPDATE_LIFETIME)) {
+ lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
+ }
+ String queue = null;
+ if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
+ queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
+ }
+ String[] nameAndFile = cliParser.getOptionValues(SAVE_CMD);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionSave(nameAndFile[1], nameAndFile[0], lifetime, queue);
+ } else if (cliParser.hasOption(DESTROY_CMD)) {
+ if (hasAnyOtherCLIOptions(cliParser, opts, DESTROY_CMD, APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .actionDestroy(cliParser.getOptionValue(DESTROY_CMD));
+ } else if (cliParser.hasOption(FLEX_CMD)) {
+ if (!cliParser.hasOption(COMPONENT) ||
+ hasAnyOtherCLIOptions(cliParser, opts, FLEX_CMD, COMPONENT,
+ APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ String[] rawCounts = cliParser.getOptionValues(COMPONENT);
+ Map<String, String> counts = new HashMap<>(rawCounts.length/2);
+ for (int i = 0; i < rawCounts.length - 1; i+=2) {
+ counts.put(rawCounts[i], rawCounts[i+1]);
+ }
+ String[] appNameAndType = getAppNameAndType(cliParser, FLEX_CMD);
+ return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
+ .actionFlex(appNameAndType[0], counts);
+ } else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) {
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH,
+ APP_TYPE_CMD)) {
+ printUsage(title, opts);
+ return exitCode;
+ }
+ return AppAdminClient.createAppAdminClient(appType, getConf())
+ .enableFastLaunch();
} else if (cliParser.hasOption(UPDATE_LIFETIME)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
@@ -332,19 +529,6 @@ public class ApplicationCLI extends YarnCLI {
}
moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
- } else if (cliParser.hasOption(SIGNAL_CMD)) {
- if (args.length < 3 || args.length > 4) {
- printUsage(title, opts);
- return exitCode;
- }
- final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
- final String containerId = signalArgs[0];
- SignalContainerCommand command =
- SignalContainerCommand.OUTPUT_THREAD_DUMP;
- if (signalArgs.length == 2) {
- command = SignalContainerCommand.valueOf(signalArgs[1]);
- }
- signalToContainer(containerId, command);
} else {
syserr.println("Invalid Command Usage : ");
printUsage(title, opts);
@@ -352,6 +536,47 @@ public class ApplicationCLI extends YarnCLI {
return 0;
}
+ private ApplicationReport getApplicationReport(ApplicationId applicationId)
+ throws IOException, YarnException {
+ ApplicationReport appReport = null;
+ try {
+ appReport = client.getApplicationReport(applicationId);
+ } catch (ApplicationNotFoundException e) {
+ throw new YarnException("Application with id '" + applicationId
+ + "' doesn't exist in RM or Timeline Server.");
+ }
+ return appReport;
+ }
+
+ private String[] getAppNameAndType(CommandLine cliParser, String option)
+ throws IOException, YarnException {
+ String applicationIdOrName = cliParser.getOptionValue(option);
+ try {
+ ApplicationId id = ApplicationId.fromString(applicationIdOrName);
+ ApplicationReport report = getApplicationReport(id);
+ return new String[]{report.getName(), report.getApplicationType()};
+ } catch (IllegalArgumentException e) {
+ // assume CLI option provided the app name
+ // and read appType from command line since id wasn't provided
+ String appType = getSingleAppTypeFromCLI(cliParser);
+ return new String[]{applicationIdOrName, appType};
+ }
+ }
+
+ private static String getSingleAppTypeFromCLI(CommandLine cliParser) {
+ if (cliParser.hasOption(APP_TYPE_CMD)) {
+ String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
+ if (types != null) {
+ for (String type : types) {
+ if (!type.trim().isEmpty()) {
+ return StringUtils.toLowerCase(type).trim();
+ }
+ }
+ }
+ }
+ return AppAdminClient.DEFAULT_TYPE;
+ }
+
private void updateApplicationTimeout(String applicationId,
ApplicationTimeoutType timeoutType, long timeoutInSec)
throws YarnException, IOException {
@@ -572,7 +797,7 @@ public class ApplicationCLI extends YarnCLI {
/**
* Kills applications with the application id as appId
*
- * @param Array of applicationIds
+ * @param applicationIds Array of applicationIds
* @return errorCode
* @throws YarnException
* @throws IOException
@@ -663,10 +888,10 @@ public class ApplicationCLI extends YarnCLI {
* Prints the application report for an application id.
*
* @param applicationId
- * @return exitCode
+ * @return ApplicationReport
* @throws YarnException
*/
- private int printApplicationReport(String applicationId)
+ private ApplicationReport printApplicationReport(String applicationId)
throws YarnException, IOException {
ApplicationReport appReport = null;
try {
@@ -675,7 +900,7 @@ public class ApplicationCLI extends YarnCLI {
} catch (ApplicationNotFoundException e) {
sysout.println("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
- return -1;
+ return null;
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -739,11 +964,11 @@ public class ApplicationCLI extends YarnCLI {
+ "' doesn't exist in RM.");
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
- return -1;
+ return null;
}
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
- return 0;
+ return appReport;
}
private void printResourceUsage(PrintWriter appReportStr,
@@ -856,11 +1081,12 @@ public class ApplicationCLI extends YarnCLI {
@SuppressWarnings("unchecked")
private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts,
- String excludeOption) {
+ String... excludeOptions) {
Collection<Option> ops = opts.getOptions();
+ Set<String> excludeSet = new HashSet<>(Arrays.asList(excludeOptions));
for (Option op : ops) {
- // Skip exclude option from the option list
- if (op.getOpt().equals(excludeOption)) {
+ // Skip exclude options from the option list
+ if (excludeSet.contains(op.getOpt())) {
continue;
}
if (cliParser.hasOption(op.getOpt())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 84cfb0a..da1659f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2025,49 +2025,121 @@ public class TestYarnCLI {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: application");
- pw.println(" -appId <Application ID> Specify Application Id to be operated");
- pw.println(" -appStates <States> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application states. The valid application");
- pw.println(" state can be one of the following:");
- pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
- pw.println(" NING,FINISHED,FAILED,KILLED");
- pw.println(" -appTags <Tags> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application tags.");
- pw.println(" -appTypes <Types> Works with -list to filter applications");
- pw.println(" based on input comma-separated list of");
- pw.println(" application types.");
- pw.println(" -changeQueue <Queue Name> Moves application to a new queue.");
- pw.println(" ApplicationId can be passed using 'appId'");
- pw.println(" option. 'movetoqueue' command is");
- pw.println(" deprecated, this new command");
- pw.println(" 'changeQueue' performs same");
- pw.println(" functionality.");
- pw.println(" -help Displays help for all commands.");
- pw.println(" -kill <Application ID> Kills the application. Set of");
- pw.println(" applications can be provided separated");
- pw.println(" with space");
- pw.println(" -list List applications. Supports optional use");
- pw.println(" of -appTypes to filter applications based");
- pw.println(" on application type, -appStates to filter");
- pw.println(" applications based on application state");
- pw.println(" and -appTags to filter applications based");
- pw.println(" on application tag.");
- pw.println(" -movetoqueue <Application ID> Moves the application to a different");
- pw.println(" queue. Deprecated command. Use");
- pw.println(" 'changeQueue' instead.");
- pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
- pw.println(" specify which queue to move an");
- pw.println(" application to.");
- pw.println(" -status <Application ID> Prints the status of the application.");
- pw.println(" -updateLifetime <Timeout> update timeout of an application from");
- pw.println(" NOW. ApplicationId can be passed using");
- pw.println(" 'appId' option. Timeout value is in");
- pw.println(" seconds.");
- pw.println(" -updatePriority <Priority> update priority of an application.");
- pw.println(" ApplicationId can be passed using 'appId'");
- pw.println(" option.");
+ pw.println(" -appId <Application ID> Specify Application Id to be");
+ pw.println(" operated");
+ pw.println(" -appStates <States> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application states. The valid");
+ pw.println(" application state can be one of");
+ pw.println(" the following:");
+ pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACC");
+ pw.println(" EPTED,RUNNING,FINISHED,FAILED,KI");
+ pw.println(" LLED");
+ pw.println(" -appTags <Tags> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application tags.");
+ pw.println(" -appTypes <Types> Works with -list to filter");
+ pw.println(" applications based on input");
+ pw.println(" comma-separated list of");
+ pw.println(" application types.");
+ pw.println(" -changeQueue <Queue Name> Moves application to a new");
+ pw.println(" queue. ApplicationId can be");
+ pw.println(" passed using 'appId' option.");
+ pw.println(" 'movetoqueue' command is");
+ pw.println(" deprecated, this new command");
+ pw.println(" 'changeQueue' performs same");
+ pw.println(" functionality.");
+ pw.println(" -component <Component Name> <Count> Works with -flex option to");
+ pw.println(" change the number of");
+ pw.println(" components/containers running");
+ pw.println(" for an application /");
+ pw.println(" long-running service. Supports");
+ pw.println(" absolute or relative changes,");
+ pw.println(" such as +1, 2, or -3.");
+ pw.println(" -destroy <Application Name> Destroys a saved application");
+ pw.println(" specification and removes all");
+ pw.println(" application data permanently.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -enableFastLaunch Uploads AM dependencies to HDFS");
+ pw.println(" to make future launches faster.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -flex <Application Name or ID> Changes number of running");
+ pw.println(" containers for a component of an");
+ pw.println(" application / long-running");
+ pw.println(" service. Requires -component");
+ pw.println(" option. If name is provided,");
+ pw.println(" appType must be provided unless");
+ pw.println(" it is the default yarn-service.");
+ pw.println(" If ID is provided, the appType");
+ pw.println(" will be looked up. Supports");
+ pw.println(" -appTypes option to specify");
+ pw.println(" which client implementation to");
+ pw.println(" use.");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -kill <Application ID> Kills the application. Set of");
+ pw.println(" applications can be provided");
+ pw.println(" separated with space");
+ pw.println(" -launch <Application Name> <File Name> Launches application from");
+ pw.println(" specification file (saves");
+ pw.println(" specification and starts");
+ pw.println(" application). Options");
+ pw.println(" -updateLifetime and -changeQueue");
+ pw.println(" can be specified to alter the");
+ pw.println(" values provided in the file.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -list List applications. Supports");
+ pw.println(" optional use of -appTypes to");
+ pw.println(" filter applications based on");
+ pw.println(" application type, -appStates to");
+ pw.println(" filter applications based on");
+ pw.println(" application state and -appTags");
+ pw.println(" to filter applications based on");
+ pw.println(" application tag.");
+ pw.println(" -movetoqueue <Application ID> Moves the application to a");
+ pw.println(" different queue. Deprecated");
+ pw.println(" command. Use 'changeQueue'");
+ pw.println(" instead.");
+ pw.println(" -queue <Queue Name> Works with the movetoqueue");
+ pw.println(" command to specify which queue");
+ pw.println(" to move an application to.");
+ pw.println(" -save <Application Name> <File Name> Saves specification file for an");
+ pw.println(" application. Options");
+ pw.println(" -updateLifetime and -changeQueue");
+ pw.println(" can be specified to alter the");
+ pw.println(" values provided in the file.");
+ pw.println(" Supports -appTypes option to");
+ pw.println(" specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -start <Application Name> Starts a previously saved");
+ pw.println(" application. Supports -appTypes");
+ pw.println(" option to specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -status <Application ID> Prints the status of the");
+ pw.println(" application.");
+ pw.println(" -stop <Application Name or ID> Stops application gracefully");
+ pw.println(" (may be started again later). If");
+ pw.println(" name is provided, appType must");
+ pw.println(" be provided unless it is the");
+ pw.println(" default yarn-service. If ID is");
+ pw.println(" provided, the appType will be");
+ pw.println(" looked up. Supports -appTypes");
+ pw.println(" option to specify which client");
+ pw.println(" implementation to use.");
+ pw.println(" -updateLifetime <Timeout> update timeout of an application");
+ pw.println(" from NOW. ApplicationId can be");
+ pw.println(" passed using 'appId' option.");
+ pw.println(" Timeout value is in seconds.");
+ pw.println(" -updatePriority <Priority> update priority of an");
+ pw.println(" application. ApplicationId can");
+ pw.println(" be passed using 'appId' option.");
pw.close();
String appsHelpStr = baos.toString("UTF-8");
return appsHelpStr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 6f00c5c..71c9e80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -37,25 +37,36 @@ User Commands
Commands useful for users of a Hadoop cluster.
-### `application`
+### `application` or `app`
Usage: `yarn application [options] `
+Usage: `yarn app [options] `
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -appId \<ApplicationId\> | Specify Application Id to be operated |
| -appStates \<States\> | Works with -list to filter applications based on input comma-separated list of application states. The valid application state can be one of the following: ALL, NEW, NEW\_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED |
+| -appTags \<Tags\> | Works with -list to filter applications based on input comma-separated list of application tags. |
| -appTypes \<Types\> | Works with -list to filter applications based on input comma-separated list of application types. |
-| -help | Help |
-| -list | Lists applications from the RM. Supports optional use of -appTypes to filter applications based on application type, and -appStates to filter applications based on application state. |
-| -kill \<ApplicationId\> | Kills the application. |
-| -movetoqueue \<Application Id\> | Moves the application to a different queue. |
+| -changeQueue \<Queue Name\> | Moves application to a new queue. ApplicationId can be passed using 'appId' option. 'movetoqueue' command is deprecated, this new command 'changeQueue' performs same functionality. |
+| -component \<Component Name\> \<Count\> | Works with -flex option to change the number of components/containers running for an application / long-running service. Supports absolute or relative changes, such as +1, 2, or -3. |
+| -destroy \<Application Name\> | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. |
+| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. |
+| -flex \<Application Name or ID\> | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
+| -help | Displays help for all commands. |
+| -kill \<Application ID\> | Kills the application. Set of applications can be provided separated with space |
+| -launch \<Application Name\> \<File Name\> | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
+| -list | List applications. Supports optional use of -appTypes to filter applications based on application type, -appStates to filter applications based on application state and -appTags to filter applications based on application tag. |
+| -movetoqueue \<Application ID\> | Moves the application to a different queue. Deprecated command. Use 'changeQueue' instead. |
| -queue \<Queue Name\> | Works with the movetoqueue command to specify which queue to move an application to. |
+| -save \<Application Name\> \<File Name\> | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
+| -start \<Application Name\> | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. |
| -status \<ApplicationId\> | Prints the status of the application. |
-| -updateLifetime \<Timeout\> | Update application timeout (from the time of request) in seconds. ApplicationId can be specified using 'appId' option. |
+| -stop \<Application Name or ID\> | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
+| -updateLifetime \<Timeout\> | Update timeout of an application from NOW. ApplicationId can be passed using 'appId' option. Timeout value is in seconds. |
| -updatePriority \<Priority\> | Update priority of an application. ApplicationId can be passed using 'appId' option. |
-Prints application(s) report/kill application
+Prints application(s) report/kill application/manage long running application
### `applicationattempt`
@@ -69,61 +80,6 @@ Usage: `yarn applicationattempt [options] `
prints applicationattempt(s) report
-### `service`
-Usage `yarn service [sub-command] [service-name] [options]`
-
-#### `SUB_COMMAND`
-
-* `build`: Build a service with its specifications, but do not start it.
- ```
- Usage: yarn service build [service-name] --file [file]
- Fields:
- service-name Optional. If specified, it will override the name in the service definition.
-
- Options:
- --file,-f The local path to the service definition file
- ```
-
-* `create`: create a service, it's equivalent to first invoke build and then start.
- ```
- Usage: yarn service create [service-name] --file [file]
- Fields:
- service-name Optional. If specified, it will override the name in the service definition.
-
- Options:
- --file,-f The local path to the service definition file.
- --queue,-q The queue to which the service is submitted.
- --example,-e The name of the example service such as:
- Sleeper A simple service that launches a few non-docker sleep containers on YARN.
- ```
-
-* `dependency`: Yarn service framework dependency (libraries) management.
- ```
- Usage: yarn service dependency [options]
- Option:
- --upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
- ```
-
-* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
- ```
- Usage: yarn service destroy [service-name]
- ```
-* `flex`: Flex a service's component by increasing or decreasing the number of containers.
- ```
- Usage: yarn service flex [service-name] --component [component-name] [count]
- Options:
- --component [component-name] [count]
- Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
- ```
-* `status`: Get the status of a service.
- ```
- Usage: yarn service status [service-name]
- ```
-* `start`: Start a service with pre-built specification or a previously stopped service.
- ```
- Usage: yarn service start [service-name]
- ```
-
### `classpath`
Usage: `yarn classpath [--glob |--jar <path> |-h |--help]`
@@ -328,12 +284,6 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
-### apiserver
-
-Usage: `yarn apiserver`
-
-Start the API-server for deploying/managing services on YARN
-
### registrydns
Usage: `yarn registrydns`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index db6d553..e4881ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -100,16 +100,18 @@ ProxyPassReverse "/" "balancer://test/"
Then run the service with the command:
```
-yarn service create [service-name] --example httpd
+yarn app -launch <service-name> httpd
```
-where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
+
+The last argument is either the path to a JSON specification of the service, or in this case, the name of an example service.
+The directory where examples can be found can be configured by setting the YARN\_EXAMPLES\_DIR environment variable.
Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page.
The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!"
The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`.
-If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+If unsure of your hostnames, visit the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
## Apache web server - httpd (without registry DNS)
@@ -151,9 +153,9 @@ hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examp
Then run the service with the command:
```
-yarn service create [service-name] --example httpd-no-dns
+yarn app -launch <service-name> httpd-no-dns
```
where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
-Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
+Look up your IPs at the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
Then visit port 8080 for each IP to view the pages.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index afcd5f5..512c011 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -45,11 +45,11 @@ Below is a simple service definition that launches sleep containers on YARN by w
```
User can simply run a pre-built example service on YARN using below command:
```
-yarn service create [service-name] --example [example-name]
+yarn app -launch <service-name> <example-name>
```
e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
-yarn service create my-sleeper --example sleeper
+yarn app -launch my-sleeper sleeper
```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
@@ -58,51 +58,53 @@ Below steps walk you through deploying a services on YARN using CLI.
Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
### Deploy a service
```
-yarn service create --file ${PATH_TO_SERVICE_DEF_FILE}
+yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE}
```
Params:
-- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services.
+- SERVICE_NAME: The name of the service. Note that this needs to be unique across running services for the current user.
- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format.
For example:
```
-yarn service create --file /path/to/local/sleeper.json
+yarn app -launch sleeper-service /path/to/local/sleeper.json
```
### Flex a component of a service
Increase or decrease the number of containers for a component.
```
-yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
+yarn app -flex ${SERVICE_NAME} -component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
```
For example, for a service named `sleeper-service`:
Set the `sleeper` component to `2` containers (absolute number).
```
-yarn service flex sleeper-service --component sleeper 2
+yarn app -flex sleeper-service -component sleeper 2
```
+Relative changes are also supported for the ${NUMBER_OF_CONTAINERS} in the flex command, such as +2 or -2.
+
### Stop a service
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
-yarn service stop ${SERVICE_NAME}
+yarn app -stop ${SERVICE_NAME}
```
### Restart a stopped service
Restarting a stopped service is easy - just call start!
```
-yarn service start ${SERVICE_NAME}
+yarn app -start ${SERVICE_NAME}
```
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
-yarn service destroy ${SERVICE_NAME}
+yarn app -destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
-YARN API Server REST API can be activated in two modes: embedded or standalone.
+YARN API Server REST API can be activated as part of the ResourceManager.
### Start Embedded API-Server as part of ResourceManager
For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
@@ -119,13 +121,6 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res
Services can be deployed on YARN through the ResourceManager web endpoint.
-### Start Standalone API-Server for deploying services on YARN
-API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
-```
- yarn --daemon start apiserver
- ```
-The above command starts the API Server on the localhost at port 9191 by default.
-
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Deploy a service
@@ -135,19 +130,14 @@ POST the aforementioned example service definition to the ResourceManager api-se
POST http://localhost:8088/ws/v1/services
```
-Or standalone API server:
-```
-POST http://localhost:9191/ws/v1/services
-```
-
### Get a service status
```
-GET http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+GET http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
### Flex a component of a service
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
```
`PUT` Request body:
```
@@ -168,7 +158,7 @@ For example:
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@@ -183,7 +173,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
Restarting a stopped service is easy.
```
-PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@@ -196,7 +186,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
-DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME}
+DELETE http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
## Services UI with YARN UI2 and Timeline Service v2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index e9aaf7e..7a2490e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -37,7 +37,7 @@ License URL: http://www.apache.org/licenses/LICENSE-2.0.html
### URI scheme
Host: host.mycompany.com
-Port: 9191(default)
+Port: 8088(default RM port)
Schemes: HTTP
@@ -368,7 +368,7 @@ The current status of a submitted service, returned as a response to the GET API
## Examples
### Create a simple single-component service with most attribute values as defaults
-POST URL - http://localhost:9191/ws/v1/services
+POST URL - http://localhost:8088/ws/v1/services
##### POST Request JSON
```json
@@ -394,7 +394,7 @@ POST URL - http://localhost:9191/ws/v1/services
```
##### GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
+GET URL - http://localhost:8088/ws/v1/services/hello-world
Note, lifetime value of -1 means unlimited lifetime.
@@ -452,7 +452,7 @@ Note, lifetime value of -1 means unlimited lifetime.
```
### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
@@ -464,7 +464,7 @@ Note, irrespective of what the current lifetime value is, this update request wi
}
```
### Stop a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@@ -474,7 +474,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Start a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@@ -484,7 +484,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Update to flex up/down the no of containers (instances) of a component of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
+PUT URL - http://localhost:8088/ws/v1/services/hello-world/components/hello
##### PUT Request JSON
```json
@@ -495,12 +495,12 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
```
### Destroy a service
-DELETE URL - http://localhost:9191/ws/v1/services/hello-world
+DELETE URL - http://localhost:8088/ws/v1/services/hello-world
***
### Create a complicated service - HBase
-POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
+POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
##### POST Request JSON
@@ -595,4 +595,4 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
"Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
}
}
-```
\ No newline at end of file
+```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-7458.
TestContainerManagerSecurity is still flakey (Contributed by Robert Kanter
via Daniel Templeton)
Posted by su...@apache.org.
YARN-7458. TestContainerManagerSecurity is still flakey
(Contributed by Robert Kanter via Daniel Templeton)
Change-Id: Ibb1975ad086c3a33f8af0b4f8b9a13c3cdca3f7d
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49b4c0b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49b4c0b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49b4c0b3
Branch: refs/heads/YARN-5881
Commit: 49b4c0b334e5472dbbf71b042a6a6b1d4b2ce3b7
Parents: 0de1068
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Nov 8 17:31:14 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Nov 8 17:31:14 2017 -0800
----------------------------------------------------------------------
.../server/TestContainerManagerSecurity.java | 38 ++++++++++++--------
1 file changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49b4c0b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 1cbad70..ad2f68a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -28,7 +28,9 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
+import java.util.concurrent.TimeoutException;
+import com.google.common.base.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputBuffer;
@@ -36,6 +38,7 @@ import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
@@ -49,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -404,27 +408,33 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
newContainerToken, attempt1NMToken, false).isEmpty());
}
- private void waitForContainerToFinishOnNM(ContainerId containerId) {
+ private void waitForContainerToFinishOnNM(ContainerId containerId)
+ throws TimeoutException, InterruptedException {
Context nmContext = yarnCluster.getNodeManager(0).getNMContext();
int interval = 4 * 60; // Max time for container token to expire.
- Assert.assertNotNull(nmContext.getContainers().containsKey(containerId));
-
- // Get the container first, as it may be removed from the Context
- // by asynchronous calls.
- // This was leading to a flakey test as otherwise the container could
- // be removed and end up null.
+ // If the container is null, then it has already completed and been removed
+ // from the Context by asynchronous calls.
Container waitContainer = nmContext.getContainers().get(containerId);
-
- while ((interval-- > 0)
- && !waitContainer.cloneAndGetContainerStatus()
- .getState().equals(ContainerState.COMPLETE)) {
+ if (waitContainer != null) {
try {
- LOG.info("Waiting for " + containerId + " to complete.");
- Thread.sleep(1000);
- } catch (InterruptedException e) {
+ LOG.info("Waiting for " + containerId + " to get to state " +
+ ContainerState.COMPLETE);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return ContainerState.COMPLETE.equals(
+ waitContainer.cloneAndGetContainerStatus().getState());
+ }
+ }, 10, interval);
+ } catch (TimeoutException te) {
+ fail("Was waiting for " + containerId + " to get to state " +
+ ContainerState.COMPLETE + " but was in state " +
+ waitContainer.cloneAndGetContainerStatus().getState() +
+ " after the timeout");
}
}
+
// Normally, Containers will be removed from NM context after they are
// explicitly acked by RM. Now, manually remove it for testing.
yarnCluster.getNodeManager(0).getNodeStatusUpdater()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-15018. Update JAVA_HOME in
create-release for Xenial Dockerfile.
Posted by su...@apache.org.
HADOOP-15018. Update JAVA_HOME in create-release for Xenial Dockerfile.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51e882d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51e882d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51e882d5
Branch: refs/heads/YARN-5881
Commit: 51e882d5c9fd2f55cd9ac2eafd3e59eb4f239d9d
Parents: 8db9d61
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Nov 7 16:38:53 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Nov 7 16:39:04 2017 -0800
----------------------------------------------------------------------
dev-support/bin/create-release | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51e882d5/dev-support/bin/create-release
----------------------------------------------------------------------
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index b98c058..694820b 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -489,9 +489,9 @@ function dockermode
echo "RUN mkdir -p /maven"
echo "RUN chown -R ${user_name} /maven"
- # we always force build with the Oracle JDK
+ # we always force build with the OpenJDK JDK
# but with the correct version
- echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-oracle"
+ echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
echo "USER ${user_name}"
printf "\n\n"
) | docker build -t "${imgname}" -
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-7413. Support resource type
in SLS (Contributed by Yufei Gu via Daniel Templeton)
Posted by su...@apache.org.
YARN-7413. Support resource type in SLS (Contributed by Yufei Gu via Daniel Templeton)
Change-Id: Ic0a897c123c5d2f57aae757ca6bcf1dad7b90d2b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba813661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba813661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba813661
Branch: refs/heads/YARN-5881
Commit: ba8136615ab66c450884614557eddc6509d63b7c
Parents: 462f6c4
Author: Daniel Templeton <te...@apache.org>
Authored: Thu Nov 9 12:09:48 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu Nov 9 12:09:48 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 86 +++++++++++++-------
.../hadoop/yarn/sls/conf/SLSConfiguration.java | 6 +-
.../yarn/sls/nodemanager/NMSimulator.java | 15 ++--
.../src/site/markdown/SchedulerLoadSimulator.md | 48 +++++++++++
.../yarn/sls/nodemanager/TestNMSimulator.java | 3 +-
5 files changed, 117 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 9d6c3aa..ad4310f 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -84,6 +85,7 @@ import org.apache.hadoop.yarn.sls.synthetic.SynthJob;
import org.apache.hadoop.yarn.sls.synthetic.SynthTraceJobProducer;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,7 +101,7 @@ public class SLSRunner extends Configured implements Tool {
// NM simulator
private HashMap<NodeId, NMSimulator> nmMap;
- private int nmMemoryMB, nmVCores;
+ private Resource nodeManagerResource;
private String nodeFile;
// AM simulator
@@ -178,6 +180,30 @@ public class SLSRunner extends Configured implements Tool {
amClassMap.put(amType, Class.forName(tempConf.get(key)));
}
}
+
+ nodeManagerResource = getNodeManagerResource();
+ }
+
+ private Resource getNodeManagerResource() {
+ Resource resource = Resources.createResource(0);
+ ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+ for (ResourceInformation info : infors) {
+ long value;
+ if (info.getName().equals(ResourceInformation.MEMORY_URI)) {
+ value = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
+ SLSConfiguration.NM_MEMORY_MB_DEFAULT);
+ } else if (info.getName().equals(ResourceInformation.VCORES_URI)) {
+ value = getConf().getInt(SLSConfiguration.NM_VCORES,
+ SLSConfiguration.NM_VCORES_DEFAULT);
+ } else {
+ value = getConf().getLong(SLSConfiguration.NM_PREFIX +
+ info.getName(), SLSConfiguration.NM_RESOURCE_DEFAULT);
+ }
+
+ resource.setResourceValue(info.getName(), value);
+ }
+
+ return resource;
}
/**
@@ -261,10 +287,6 @@ public class SLSRunner extends Configured implements Tool {
private void startNM() throws YarnException, IOException {
// nm configuration
- nmMemoryMB = getConf().getInt(SLSConfiguration.NM_MEMORY_MB,
- SLSConfiguration.NM_MEMORY_MB_DEFAULT);
- nmVCores = getConf().getInt(SLSConfiguration.NM_VCORES,
- SLSConfiguration.NM_VCORES_DEFAULT);
int heartbeatInterval =
getConf().getInt(SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS,
SLSConfiguration.NM_HEARTBEAT_INTERVAL_MS_DEFAULT);
@@ -304,7 +326,7 @@ public class SLSRunner extends Configured implements Tool {
for (String hostName : nodeSet) {
// we randomize the heartbeat start time from zero to 1 interval
NMSimulator nm = new NMSimulator();
- nm.init(hostName, nmMemoryMB, nmVCores, random.nextInt(heartbeatInterval),
+ nm.init(hostName, nodeManagerResource, random.nextInt(heartbeatInterval),
heartbeatInterval, rm);
nmMap.put(nm.getNode().getNodeID(), nm);
runner.schedule(nm);
@@ -460,18 +482,7 @@ public class SLSRunner extends Configured implements Tool {
+ " to 0!");
}
- Resource res = getDefaultContainerResource();
- if (jsonTask.containsKey(SLSConfiguration.TASK_MEMORY)) {
- int containerMemory = Integer.parseInt(
- jsonTask.get(SLSConfiguration.TASK_MEMORY).toString());
- res.setMemorySize(containerMemory);
- }
-
- if (jsonTask.containsKey(SLSConfiguration.CONTAINER_VCORES)) {
- int containerVCores = Integer.parseInt(
- jsonTask.get(SLSConfiguration.CONTAINER_VCORES).toString());
- res.setVirtualCores(containerVCores);
- }
+ Resource res = getResourceForContainer(jsonTask);
int priority = DEFAULT_MAPPER_PRIORITY;
if (jsonTask.containsKey(SLSConfiguration.TASK_PRIORITY)) {
@@ -500,6 +511,21 @@ public class SLSRunner extends Configured implements Tool {
return containers;
}
+ private Resource getResourceForContainer(Map jsonTask) {
+ Resource res = getDefaultContainerResource();
+ ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+ for (ResourceInformation info : infors) {
+ if (jsonTask.containsKey(SLSConfiguration.TASK_PREFIX + info.getName())) {
+ long value = Long.parseLong(
+ jsonTask.get(SLSConfiguration.TASK_PREFIX + info.getName())
+ .toString());
+ res.setResourceValue(info.getName(), value);
+ }
+ }
+
+ return res;
+ }
+
/**
* Parse workload from a rumen trace file.
*/
@@ -717,15 +743,15 @@ public class SLSRunner extends Configured implements Tool {
return amContainerResource;
}
- if (jsonJob.containsKey(SLSConfiguration.AM_MEMORY)) {
- amContainerResource.setMemorySize(
- Long.parseLong(jsonJob.get(SLSConfiguration.AM_MEMORY).toString()));
+ ResourceInformation[] infors = ResourceUtils.getResourceTypesArray();
+ for (ResourceInformation info : infors) {
+ String key = SLSConfiguration.JOB_AM_PREFIX + info.getName();
+ if (jsonJob.containsKey(key)) {
+ long value = Long.parseLong(jsonJob.get(key).toString());
+ amContainerResource.setResourceValue(info.getName(), value);
+ }
}
- if (jsonJob.containsKey(SLSConfiguration.AM_VCORES)) {
- amContainerResource.setVirtualCores(
- Integer.parseInt(jsonJob.get(SLSConfiguration.AM_VCORES).toString()));
- }
return amContainerResource;
}
@@ -777,8 +803,8 @@ public class SLSRunner extends Configured implements Tool {
// node
LOG.info("------------------------------------");
LOG.info("# nodes = {}, # racks = {}, capacity " +
- "of each node {} MB memory and {} vcores.",
- numNMs, numRacks, nmMemoryMB, nmVCores);
+ "of each node {}.",
+ numNMs, numRacks, nodeManagerResource);
LOG.info("------------------------------------");
// job
LOG.info("# applications = {}, # total " +
@@ -804,8 +830,10 @@ public class SLSRunner extends Configured implements Tool {
// package these information in the simulateInfoMap used by other places
simulateInfoMap.put("Number of racks", numRacks);
simulateInfoMap.put("Number of nodes", numNMs);
- simulateInfoMap.put("Node memory (MB)", nmMemoryMB);
- simulateInfoMap.put("Node VCores", nmVCores);
+ simulateInfoMap.put("Node memory (MB)",
+ nodeManagerResource.getResourceValue(ResourceInformation.MEMORY_URI));
+ simulateInfoMap.put("Node VCores",
+ nodeManagerResource.getResourceValue(ResourceInformation.VCORES_URI));
simulateInfoMap.put("Number of applications", numAMs);
simulateInfoMap.put("Number of tasks", numTasks);
simulateInfoMap.put("Average tasks per applicaion",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
index 58f0c8c..7fc2a3c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java
@@ -54,6 +54,7 @@ public class SLSConfiguration {
public static final int NM_MEMORY_MB_DEFAULT = 10240;
public static final String NM_VCORES = NM_PREFIX + "vcores";
public static final int NM_VCORES_DEFAULT = 10;
+ public static final int NM_RESOURCE_DEFAULT = 0;
public static final String NM_HEARTBEAT_INTERVAL_MS = NM_PREFIX
+ "heartbeat.interval.ms";
public static final int NM_HEARTBEAT_INTERVAL_MS_DEFAULT = 1000;
@@ -65,12 +66,10 @@ public class SLSConfiguration {
public static final String AM_TYPE = AM_PREFIX + "type";
public static final String AM_TYPE_PREFIX = AM_TYPE + ".";
- public static final String AM_MEMORY = AM_PREFIX + "memory";
public static final String AM_CONTAINER_MEMORY = AM_PREFIX +
"container.memory";
public static final int AM_CONTAINER_MEMORY_DEFAULT = 1024;
- public static final String AM_VCORES = AM_PREFIX + "vcores";
public static final String AM_CONTAINER_VCORES = AM_PREFIX +
"container.vcores";
public static final int AM_CONTAINER_VCORES_DEFAULT = 1;
@@ -104,8 +103,10 @@ public class SLSConfiguration {
public static final String JOB_USER = JOB_PREFIX + "user";
public static final String JOB_COUNT = JOB_PREFIX + "count";
public static final String JOB_TASKS = JOB_PREFIX + "tasks";
+ public static final String JOB_AM_PREFIX = "am.";
// task
+ public static final String TASK_PREFIX = "container.";
public static final String COUNT = "count";
public static final String TASK_CONTAINER = "container.";
public static final String TASK_HOST = TASK_CONTAINER + "host";
@@ -115,6 +116,5 @@ public class SLSConfiguration {
public static final String TASK_DURATION_MS = TASK_CONTAINER + DURATION_MS;
public static final String TASK_PRIORITY = TASK_CONTAINER + "priority";
public static final String TASK_TYPE = TASK_CONTAINER + "type";
- public static final String TASK_MEMORY = TASK_CONTAINER + "memory";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
index 9197b1e..6b19128 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
@@ -35,24 +35,23 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords
- .RegisterNodeManagerRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords
- .RegisterNodeManagerResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,15 +74,15 @@ public class NMSimulator extends TaskRunner.Task {
private int RESPONSE_ID = 1;
private final static Logger LOG = LoggerFactory.getLogger(NMSimulator.class);
- public void init(String nodeIdStr, int memory, int cores,
+ public void init(String nodeIdStr, Resource nodeResource,
int dispatchTime, int heartBeatInterval, ResourceManager rm)
throws IOException, YarnException {
super.init(dispatchTime, dispatchTime + 1000000L * heartBeatInterval,
heartBeatInterval);
// create resource
String rackHostName[] = SLSUtils.getRackHostName(nodeIdStr);
- this.node = NodeInfo.newNodeInfo(rackHostName[0], rackHostName[1],
- BuilderUtils.newResource(memory, cores));
+ this.node = NodeInfo.newNodeInfo(rackHostName[0], rackHostName[1],
+ Resources.clone(nodeResource));
this.rm = rm;
// init data structures
completedContainerList =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
index 0bab9a6..9df4998 100644
--- a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -315,6 +315,54 @@ The SYNTH mode of SLS is very convenient to generate very large loads without th
files. This allows to easily explore wide range of use cases (e.g., imagine simulating 100k jobs, and in different
runs simply tune the average number of mappers, or average task duration), in an efficient and compact way.
+Resource Type in SLS
+------------------------
+This section talks about how to use resource type in SLS.
+
+## Configure Resource Manager
+This is the same to how to configure resource type for a real cluster. Configure
+item `yarn.resource-types` in yarn-site.xml as the following example does.
+
+ <property>
+ <name>yarn.resource-types</name>
+ <value>resource-type1, resource-type2</value>
+ </property>
+
+## Configure Node Manager
+Specify the size of resource in each node by adding relevant items into
+sls-runner.xml like the following example does. The values apply for every node
+in SLS. The default values for resources other than memory and vcores are 0.
+
+ <property>
+ <name>yarn.sls.nm.resource-type1</name>
+ <value>10</value>
+ </property>
+ <property>
+ <name>yarn.sls.nm.resource-type2</name>
+ <value>10</value>
+ </property>
+
+## Specify Resource in SLS JSON input
+Resource Type is supported in SLS JSON input format, but not in other two
+formats(SYNTH and RUMEN). To make it work in SLS JSON input format, you can
+specify resource sizes for both task containers and the AM container. Here is an
+example.
+
+ {
+ "job.start.ms" : 0,
+ "am.memory-mb": 2048,
+ "am.vcores": 2,
+ "am.resource-type1": 2,
+ "am.resource-type2": 2,
+ "job.tasks" : [ {
+ "container.duration.ms": 5000
+ "container.memory-mb": 1024,
+ "container.vcores": 1,
+ "container.resource-type1": 1,
+ "container.resource-type2": 1
+ }
+ }
+
Appendix
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba813661/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
index 2f10f7d..5064ef2 100644
--- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
@@ -75,7 +75,8 @@ public class TestNMSimulator {
public void testNMSimulator() throws Exception {
// Register one node
NMSimulator node1 = new NMSimulator();
- node1.init("/rack1/node1", GB * 10, 10, 0, 1000, rm);
+ node1.init("/rack1/node1", Resources.createResource(GB * 10, 10), 0, 1000,
+ rm);
node1.middleStep();
int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-7401. Reduce lock contention
in ClusterNodeTracker#getClusterCapacity()
Posted by su...@apache.org.
YARN-7401. Reduce lock contention in ClusterNodeTracker#getClusterCapacity()
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8db9d61a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8db9d61a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8db9d61a
Branch: refs/heads/YARN-5881
Commit: 8db9d61ac2e04888cb228b29fe54b41c730cf0e6
Parents: 13fa2d4
Author: Daniel Templeton <te...@apache.org>
Authored: Tue Nov 7 14:53:48 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Nov 7 14:53:48 2017 -0800
----------------------------------------------------------------------
.../scheduler/ClusterNodeTracker.java | 18 ++++++------------
1 file changed, 6 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8db9d61a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index ccec6bc..60ef390 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -55,8 +55,9 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Map<String, N> nodeNameToNodeMap = new HashMap<>();
private Map<String, List<N>> nodesPerRack = new HashMap<>();
- private Resource clusterCapacity = Resources.clone(Resources.none());
- private Resource staleClusterCapacity = null;
+ private final Resource clusterCapacity = Resources.clone(Resources.none());
+ private volatile Resource staleClusterCapacity =
+ Resources.clone(Resources.none());
// Max allocation
private long maxNodeMemory = -1;
@@ -82,6 +83,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
// Update cluster capacity
Resources.addTo(clusterCapacity, node.getTotalResource());
+ staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
updateMaxResources(node, true);
@@ -139,16 +141,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
}
public Resource getClusterCapacity() {
- readLock.lock();
- try {
- if (staleClusterCapacity == null ||
- !Resources.equals(staleClusterCapacity, clusterCapacity)) {
- staleClusterCapacity = Resources.clone(clusterCapacity);
- }
- return staleClusterCapacity;
- } finally {
- readLock.unlock();
- }
+ return staleClusterCapacity;
}
public N removeNode(NodeId nodeId) {
@@ -175,6 +168,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
// Update cluster capacity
Resources.subtractFrom(clusterCapacity, node.getTotalResource());
+ staleClusterCapacity = Resources.clone(clusterCapacity);
// Update maximumAllocation
updateMaxResources(node, false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-12788. Reset the upload
button when file upload fails. Contributed by Brahma Reddy Battula
Posted by su...@apache.org.
HDFS-12788. Reset the upload button when file upload fails. Contributed by Brahma Reddy Battula
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/410d0319
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/410d0319
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/410d0319
Branch: refs/heads/YARN-5881
Commit: 410d0319cf72b9b5f3807c522237f52121d98cd5
Parents: bb8a6ee
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Wed Nov 8 14:41:16 2017 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Wed Nov 8 14:41:16 2017 +0530
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/410d0319/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index dae3519..ed1f832 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -440,22 +440,29 @@
}).complete(function(data) {
numCompleted++;
if(numCompleted == files.length) {
- $('#modal-upload-file').modal('hide');
- $('#modal-upload-file-button').button('reset');
+ reset_upload_button();
browse_directory(current_directory);
}
}).error(function(jqXHR, textStatus, errorThrown) {
numCompleted++;
+ reset_upload_button();
show_err_msg("Couldn't upload the file " + file.file.name + ". "+ errorThrown);
});
}).error(function(jqXHR, textStatus, errorThrown) {
numCompleted++;
+ reset_upload_button();
show_err_msg("Couldn't find datanode to write file. " + errorThrown);
});
})();
}
});
+ //Reset the upload button
+ function reset_upload_button() {
+ $('#modal-upload-file').modal('hide');
+ $('#modal-upload-file-button').button('reset');
+ }
+
//Store the list of files which have been checked into session storage
function store_selected_files(current_directory) {
sessionStorage.setItem("source_directory", current_directory);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN
related params in the service REST API. Contributed by Jian He
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
index 17d38cb..0f0156a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
@@ -20,7 +20,7 @@
<div class="row">
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Detailed Information</div>
+ <div class="panel-title">{{type}} Metrics:</div>
</div>
<div class="">
<table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 5a94575..ef517d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -20,7 +20,7 @@
{{#if model.container}}
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">Component Information</div>
+ <div class="panel-title">Component Instance Information</div>
</div>
<div class="">
<table class="table table-striped table-bordered table-hover">
@@ -30,7 +30,7 @@
<td>{{check-availability model.container.instanceName}}</td>
</tr>
<tr>
- <td>Component Group</td>
+ <td>Component</td>
<td>{{check-availability model.container.component}}</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
index 0b642b0..fb4dba2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
@@ -18,7 +18,7 @@
<div class="row">
<div class="col-md-12">
- <h3>Active Components: {{model.componentName}}</h3>
+ <h3>Component: {{model.componentName}}</h3>
{{em-table columns=tableColumns rows=model.instances}}
</div>
</div>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-7331. Change in few metrics
in new YARN UI related to native-services. Contributed by Sunil G
Posted by su...@apache.org.
YARN-7331. Change in few metrics in new YARN UI related to native-services. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a288b0ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a288b0ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a288b0ac
Branch: refs/heads/YARN-5881
Commit: a288b0accf8e0df0375cd5ca3aafdbfb142aa8c3
Parents: ba7ed7b
Author: Jian He <ji...@apache.org>
Authored: Mon Oct 30 13:16:19 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../main/webapp/app/adapters/yarn-servicedef.js | 2 +-
.../app/controllers/yarn-app/components.js | 8 ++---
.../yarn-component-instances/info.js | 2 +-
.../app/serializers/yarn-component-instance.js | 2 +-
.../app/templates/components/metrics-table.hbs | 37 +++-----------------
.../templates/yarn-component-instance/info.hbs | 6 +---
.../src/main/webapp/config/configs.env | 7 ----
.../src/main/webapp/config/default-config.js | 3 +-
8 files changed, 12 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
index dc5dbfd..3fb4a81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
@@ -20,7 +20,7 @@ import Ember from 'ember';
import RESTAbstractAdapter from './restabstract';
export default RESTAbstractAdapter.extend({
- address: "dashWebAddress",
+ address: "rmWebAddress",
restNameSpace: "dashService",
serverName: "DASH",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index 947cb98..b4bac0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
cols.push({
id: 'name',
- headerTitle: 'Component Group',
+ headerTitle: 'Component',
contentPath: 'name',
cellComponentName: 'em-table-linked-cell',
getCellContent: function(row) {
@@ -49,13 +49,9 @@ export default Ember.Controller.extend({
contentPath: 'memory'
}, {
id: 'instances',
- headerTitle: '# Components',
+ headerTitle: 'Number Of Instances',
contentPath: 'instances',
observePath: true
- }, {
- id: 'createdDate',
- headerTitle: 'Created Time',
- contentPath: 'createdDate'
});
return ColumnDef.make(cols);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
index a676b34..0d1a4bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
@@ -31,7 +31,7 @@ export default Ember.Controller.extend({
cols.push({
id: 'instanceName',
- headerTitle: 'Component Name',
+ headerTitle: 'Component Instance',
contentPath: 'instanceName',
cellComponentName: 'em-table-linked-cell',
getCellContent: function(row) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
index 82eb273..1bd3b2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
@@ -28,7 +28,7 @@ export default DS.JSONAPISerializer.extend({
attributes: {
containerId: payload.id,
component: info.COMPONENT_NAME,
- instanceName: info.COMPONENT_NAME + '_' + payload.instanceId,
+ instanceName: info.COMPONENT_INSTANCE_NAME,
state: info.STATE,
createdTimestamp: payload.createdtime,
startedTimestamp: info.LAUNCH_TIME,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
index 6e4e990..17d38cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
@@ -20,7 +20,7 @@
<div class="row">
<div class="panel panel-default">
<div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Success Information</div>
+ <div class="panel-title">{{type}} Metrics: Detailed Information</div>
</div>
<div class="">
<table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
@@ -28,45 +28,18 @@
<tr>
<th>Desired Containers</th>
<th>Running Containers</th>
- <th>Completed Containers</th>
- <th>Pending Containers</th>
- <th>Surplus Containers</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td>{{metrics.ContainersDesired}}</td>
- <td>{{metrics.ContainersRunning}}</td>
- <td>{{metrics.ContainersCompleted}}</td>
- <td>{{metrics.ContainersPending}}</td>
- <td>{{metrics.SurplusContainers}}</td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
- <div class="row">
- <div class="panel panel-default">
- <div class="panel-heading">
- <div class="panel-title">{{type}} Metrics: Failure Information</div>
- </div>
- <div class="">
- <table class="table table-hover table-custom-bordered table-custom-stripped table-radius-none table-border-none">
- <thead>
- <tr>
+ <th>Ready Containers</th>
<th>Failed Containers</th>
- <th>Containers Failed Since Last Threshold</th>
<th>Preempted Containers</th>
- <th>Pending Anti-Affinity Containers</th>
</tr>
</thead>
<tbody>
<tr>
+ <td>{{metrics.ContainersDesired}}</td>
+ <td>{{metrics.ContainersRunning}}</td>
+ <td>{{metrics.ContainersReady}}</td>
<td>{{metrics.ContainersFailed}}</td>
- <td>{{metrics.FailedSinceLastThreshold}}</td>
<td>{{metrics.ContainersPreempted}}</td>
- <td>{{metrics.PendingAAContainers}}</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 3d5720e..5a94575 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -26,7 +26,7 @@
<table class="table table-striped table-bordered table-hover">
<tbody>
<tr>
- <td>Component Name</td>
+ <td>Component Instance</td>
<td>{{check-availability model.container.instanceName}}</td>
</tr>
<tr>
@@ -42,10 +42,6 @@
<td>{{check-availability model.container.state}}</td>
</tr>
<tr>
- <td>Created Time</td>
- <td>{{check-availability model.container.createdDate}}</td>
- </tr>
- <tr>
<td>Started Time</td>
<td>{{check-availability model.container.startedDate}}</td>
</tr>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
index a795fc5..04577c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/configs.env
@@ -40,13 +40,6 @@ ENV = {
*/
//rmWebAddress: "localhost:8088",
- /*
- * Dash server web interface can be configured below.
- * By default dash web address is set as localhost:9191, uncomment and change
- * the following value for pointing to a different address.
- */
- //dashWebAddress: "localhost:9191",
-
/*
* Protocol scheme. It can be "http:" or "https:". By default, http is used.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
index d24e960..792f442 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
@@ -21,7 +21,6 @@ module.exports = { // YARN UI App configurations
localBaseAddress: "",
timelineWebAddress: "localhost:8188",
rmWebAddress: "localhost:8088",
- dashWebAddress: "localhost:9191",
protocolScheme: "http:"
},
namespaces: {
@@ -30,7 +29,7 @@ module.exports = { // YARN UI App configurations
cluster: 'ws/v1/cluster',
metrics: 'ws/v1/cluster/metrics',
timelineV2: 'ws/v2/timeline',
- dashService: 'services/v1/applications',
+ dashService: 'ws/v1/services',
node: '{nodeAddress}/ws/v1/node'
},
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-7437. Rename PlacementSet and
SchedulingPlacementSet. (Wangda Tan via kkaranasos)
Posted by su...@apache.org.
YARN-7437. Rename PlacementSet and SchedulingPlacementSet. (Wangda Tan via kkaranasos)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac4d2b10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac4d2b10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac4d2b10
Branch: refs/heads/YARN-5881
Commit: ac4d2b1081d8836a21bc70e77f4e6cd2071a9949
Parents: a2c150a
Author: Konstantinos Karanasos <kk...@apache.org>
Authored: Thu Nov 9 13:01:14 2017 -0800
Committer: Konstantinos Karanasos <kk...@apache.org>
Committed: Thu Nov 9 13:01:24 2017 -0800
----------------------------------------------------------------------
.../scheduler/AppSchedulingInfo.java | 93 ++--
.../scheduler/ContainerUpdateContext.java | 15 +-
.../scheduler/SchedulerApplicationAttempt.java | 27 +-
.../scheduler/activities/ActivitiesLogger.java | 2 +-
.../scheduler/capacity/AbstractCSQueue.java | 4 +-
.../scheduler/capacity/CSQueue.java | 12 +-
.../scheduler/capacity/CapacityScheduler.java | 69 +--
.../scheduler/capacity/LeafQueue.java | 39 +-
.../scheduler/capacity/ParentQueue.java | 37 +-
.../allocator/AbstractContainerAllocator.java | 10 +-
.../capacity/allocator/ContainerAllocator.java | 11 +-
.../allocator/RegularContainerAllocator.java | 57 +--
.../scheduler/common/fica/FiCaSchedulerApp.java | 23 +-
.../scheduler/fair/FSAppAttempt.java | 2 +-
.../placement/AppPlacementAllocator.java | 163 +++++++
.../scheduler/placement/CandidateNodeSet.java | 61 +++
.../placement/CandidateNodeSetUtils.java | 44 ++
.../LocalityAppPlacementAllocator.java | 422 +++++++++++++++++++
.../LocalitySchedulingPlacementSet.java | 416 ------------------
.../scheduler/placement/PlacementSet.java | 65 ---
.../scheduler/placement/PlacementSetUtils.java | 36 --
.../placement/SchedulingPlacementSet.java | 158 -------
.../placement/SimpleCandidateNodeSet.java | 68 +++
.../scheduler/placement/SimplePlacementSet.java | 70 ---
.../scheduler/placement/package-info.java | 28 ++
.../capacity/TestCapacityScheduler.java | 5 +-
.../scheduler/capacity/TestChildQueueOrder.java | 18 +-
.../capacity/TestContainerResizing.java | 7 +-
.../TestNodeLabelContainerAllocation.java | 2 +-
.../scheduler/capacity/TestParentQueue.java | 119 +++---
30 files changed, 1082 insertions(+), 1001 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 082ec14..9f49880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -46,9 +46,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalitySchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.LocalityAppPlacementAllocator;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceRequestUpdateResult;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -82,8 +82,8 @@ public class AppSchedulingInfo {
private final ConcurrentSkipListSet<SchedulerRequestKey>
schedulerKeys = new ConcurrentSkipListSet<>();
- final Map<SchedulerRequestKey, SchedulingPlacementSet<SchedulerNode>>
- schedulerKeyToPlacementSets = new ConcurrentHashMap<>();
+ private final Map<SchedulerRequestKey, AppPlacementAllocator<SchedulerNode>>
+ schedulerKeyToAppPlacementAllocator = new ConcurrentHashMap<>();
private final ReentrantReadWriteLock.ReadLock readLock;
private final ReentrantReadWriteLock.WriteLock writeLock;
@@ -146,7 +146,7 @@ public class AppSchedulingInfo {
*/
private void clearRequests() {
schedulerKeys.clear();
- schedulerKeyToPlacementSets.clear();
+ schedulerKeyToAppPlacementAllocator.clear();
LOG.info("Application " + applicationId + " requests cleared");
}
@@ -190,9 +190,9 @@ public class AppSchedulingInfo {
dedupRequests.get(schedulerKey).put(request.getResourceName(), request);
}
- // Update scheduling placement set
+ // Update AppPlacementAllocator by dedup requests.
offswitchResourcesUpdated =
- addToPlacementSets(
+ addRequestToAppPlacement(
recoverPreemptedRequestForAContainer, dedupRequests);
return offswitchResourcesUpdated;
@@ -201,11 +201,11 @@ public class AppSchedulingInfo {
}
}
- public void removePlacementSets(SchedulerRequestKey schedulerRequestKey) {
- schedulerKeyToPlacementSets.remove(schedulerRequestKey);
+ public void removeAppPlacement(SchedulerRequestKey schedulerRequestKey) {
+ schedulerKeyToAppPlacementAllocator.remove(schedulerRequestKey);
}
- boolean addToPlacementSets(
+ boolean addRequestToAppPlacement(
boolean recoverPreemptedRequestForAContainer,
Map<SchedulerRequestKey, Map<String, ResourceRequest>> dedupRequests) {
boolean offswitchResourcesUpdated = false;
@@ -213,14 +213,15 @@ public class AppSchedulingInfo {
dedupRequests.entrySet()) {
SchedulerRequestKey schedulerRequestKey = entry.getKey();
- if (!schedulerKeyToPlacementSets.containsKey(schedulerRequestKey)) {
- schedulerKeyToPlacementSets.put(schedulerRequestKey,
- new LocalitySchedulingPlacementSet<>(this));
+ if (!schedulerKeyToAppPlacementAllocator.containsKey(
+ schedulerRequestKey)) {
+ schedulerKeyToAppPlacementAllocator.put(schedulerRequestKey,
+ new LocalityAppPlacementAllocator<>(this));
}
- // Update placement set
+ // Update AppPlacementAllocator
ResourceRequestUpdateResult pendingAmountChanges =
- schedulerKeyToPlacementSets.get(schedulerRequestKey)
+ schedulerKeyToAppPlacementAllocator.get(schedulerRequestKey)
.updateResourceRequests(
entry.getValue().values(),
recoverPreemptedRequestForAContainer);
@@ -244,7 +245,7 @@ public class AppSchedulingInfo {
if (request.getNumContainers() <= 0) {
if (lastRequestContainers >= 0) {
schedulerKeys.remove(schedulerKey);
- schedulerKeyToPlacementSets.remove(schedulerKey);
+ schedulerKeyToAppPlacementAllocator.remove(schedulerKey);
}
LOG.info("checking for deactivate of application :"
+ this.applicationId);
@@ -356,8 +357,9 @@ public class AppSchedulingInfo {
List<ResourceRequest> ret = new ArrayList<>();
try {
this.readLock.lock();
- for (SchedulingPlacementSet ps : schedulerKeyToPlacementSets.values()) {
- ret.addAll(ps.getResourceRequests().values());
+ for (AppPlacementAllocator ap : schedulerKeyToAppPlacementAllocator
+ .values()) {
+ ret.addAll(ap.getResourceRequests().values());
}
} finally {
this.readLock.unlock();
@@ -384,8 +386,9 @@ public class AppSchedulingInfo {
String resourceName) {
try {
this.readLock.lock();
- SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(schedulerKey);
- return (ps == null) ? PendingAsk.ZERO : ps.getPendingAsk(resourceName);
+ AppPlacementAllocator ap = schedulerKeyToAppPlacementAllocator.get(
+ schedulerKey);
+ return (ap == null) ? PendingAsk.ZERO : ap.getPendingAsk(resourceName);
} finally {
this.readLock.unlock();
}
@@ -424,7 +427,7 @@ public class AppSchedulingInfo {
updateMetricsForAllocatedContainer(type, node, containerAllocated);
}
- return schedulerKeyToPlacementSets.get(schedulerKey).allocate(
+ return schedulerKeyToAppPlacementAllocator.get(schedulerKey).allocate(
schedulerKey, type, node);
} finally {
writeLock.unlock();
@@ -442,23 +445,24 @@ public class AppSchedulingInfo {
this.writeLock.lock();
QueueMetrics oldMetrics = queue.getMetrics();
QueueMetrics newMetrics = newQueue.getMetrics();
- for (SchedulingPlacementSet ps : schedulerKeyToPlacementSets.values()) {
- PendingAsk ask = ps.getPendingAsk(ResourceRequest.ANY);
+ for (AppPlacementAllocator ap : schedulerKeyToAppPlacementAllocator
+ .values()) {
+ PendingAsk ask = ap.getPendingAsk(ResourceRequest.ANY);
if (ask.getCount() > 0) {
oldMetrics.decrPendingResources(
- ps.getPrimaryRequestedNodePartition(),
+ ap.getPrimaryRequestedNodePartition(),
user, ask.getCount(), ask.getPerAllocationResource());
newMetrics.incrPendingResources(
- ps.getPrimaryRequestedNodePartition(),
+ ap.getPrimaryRequestedNodePartition(),
user, ask.getCount(), ask.getPerAllocationResource());
Resource delta = Resources.multiply(ask.getPerAllocationResource(),
ask.getCount());
// Update Queue
queue.decPendingResource(
- ps.getPrimaryRequestedNodePartition(), delta);
+ ap.getPrimaryRequestedNodePartition(), delta);
newQueue.incPendingResource(
- ps.getPrimaryRequestedNodePartition(), delta);
+ ap.getPrimaryRequestedNodePartition(), delta);
}
}
oldMetrics.moveAppFrom(this);
@@ -477,15 +481,16 @@ public class AppSchedulingInfo {
try {
this.writeLock.lock();
QueueMetrics metrics = queue.getMetrics();
- for (SchedulingPlacementSet ps : schedulerKeyToPlacementSets.values()) {
- PendingAsk ask = ps.getPendingAsk(ResourceRequest.ANY);
+ for (AppPlacementAllocator ap : schedulerKeyToAppPlacementAllocator
+ .values()) {
+ PendingAsk ask = ap.getPendingAsk(ResourceRequest.ANY);
if (ask.getCount() > 0) {
- metrics.decrPendingResources(ps.getPrimaryRequestedNodePartition(),
+ metrics.decrPendingResources(ap.getPrimaryRequestedNodePartition(),
user, ask.getCount(), ask.getPerAllocationResource());
// Update Queue
queue.decPendingResource(
- ps.getPrimaryRequestedNodePartition(),
+ ap.getPrimaryRequestedNodePartition(),
Resources.multiply(ask.getPerAllocationResource(),
ask.getCount()));
}
@@ -559,11 +564,12 @@ public class AppSchedulingInfo {
SchedulerRequestKey schedulerKey) {
try {
readLock.lock();
- SchedulingPlacementSet ps = schedulerKeyToPlacementSets.get(schedulerKey);
- if (null == ps) {
+ AppPlacementAllocator ap = schedulerKeyToAppPlacementAllocator.get(
+ schedulerKey);
+ if (null == ap) {
return false;
}
- return ps.canAllocate(type, node);
+ return ap.canAllocate(type, node);
} finally {
readLock.unlock();
}
@@ -593,11 +599,10 @@ public class AppSchedulingInfo {
metrics.incrNodeTypeAggregations(user, type);
}
- // Get placement-set by specified schedulerKey
- // Now simply return all node of the input clusterPlacementSet
- public <N extends SchedulerNode> SchedulingPlacementSet<N> getSchedulingPlacementSet(
+ // Get AppPlacementAllocator by specified schedulerKey
+ public <N extends SchedulerNode> AppPlacementAllocator<N> getAppPlacementAllocator(
SchedulerRequestKey schedulerkey) {
- return (SchedulingPlacementSet<N>) schedulerKeyToPlacementSets.get(
+ return (AppPlacementAllocator<N>) schedulerKeyToAppPlacementAllocator.get(
schedulerkey);
}
@@ -614,9 +619,9 @@ public class AppSchedulingInfo {
SchedulerRequestKey schedulerKey, String resourceName) {
try {
this.readLock.lock();
- SchedulingPlacementSet ps =
- schedulerKeyToPlacementSets.get(schedulerKey);
- return (ps == null) || ps.canDelayTo(resourceName);
+ AppPlacementAllocator ap =
+ schedulerKeyToAppPlacementAllocator.get(schedulerKey);
+ return (ap == null) || ap.canDelayTo(resourceName);
} finally {
this.readLock.unlock();
}
@@ -626,9 +631,9 @@ public class AppSchedulingInfo {
String nodePartition, SchedulingMode schedulingMode) {
try {
this.readLock.lock();
- SchedulingPlacementSet ps =
- schedulerKeyToPlacementSets.get(schedulerKey);
- return (ps != null) && ps.acceptNodePartition(nodePartition,
+ AppPlacementAllocator ap =
+ schedulerKeyToAppPlacementAllocator.get(schedulerKey);
+ return (ap != null) && ap.acceptNodePartition(nodePartition,
schedulingMode);
} finally {
this.readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
index 5ac2ac5..93995a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ContainerUpdateContext.java
@@ -34,8 +34,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
.RMContainerImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
- .SchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -146,17 +145,17 @@ public class ContainerUpdateContext {
createResourceRequests(rmContainer, schedulerNode,
schedulerKey, resToIncrease);
updateResReqs.put(schedulerKey, resMap);
- appSchedulingInfo.addToPlacementSets(false, updateResReqs);
+ appSchedulingInfo.addRequestToAppPlacement(false, updateResReqs);
}
return true;
}
private void cancelPreviousRequest(SchedulerNode schedulerNode,
SchedulerRequestKey schedulerKey) {
- SchedulingPlacementSet<SchedulerNode> schedulingPlacementSet =
- appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
- if (schedulingPlacementSet != null) {
- Map<String, ResourceRequest> resourceRequests = schedulingPlacementSet
+ AppPlacementAllocator<SchedulerNode> appPlacementAllocator =
+ appSchedulingInfo.getAppPlacementAllocator(schedulerKey);
+ if (appPlacementAllocator != null) {
+ Map<String, ResourceRequest> resourceRequests = appPlacementAllocator
.getResourceRequests();
ResourceRequest prevReq = resourceRequests.get(ResourceRequest.ANY);
// Decrement the pending using a dummy RR with
@@ -290,7 +289,7 @@ public class ContainerUpdateContext {
(rmContainer, node, schedulerKey,
rmContainer.getContainer().getResource());
reqsToUpdate.put(schedulerKey, resMap);
- appSchedulingInfo.addToPlacementSets(true, reqsToUpdate);
+ appSchedulingInfo.addRequestToAppPlacement(true, reqsToUpdate);
return UNDEFINED;
}
return retVal;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index ce71afa..346bd20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import com.google.common.collect.ConcurrentHashMultiset;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang.time.FastDateFormat;
@@ -75,14 +74,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerUpdatesAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
-
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
-
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -91,6 +87,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ConcurrentHashMultiset;
/**
* Represents an application attempt from the viewpoint of the scheduler.
@@ -316,9 +313,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
String resourceName) {
try {
readLock.lock();
- SchedulingPlacementSet ps = appSchedulingInfo.getSchedulingPlacementSet(
+ AppPlacementAllocator ap = appSchedulingInfo.getAppPlacementAllocator(
schedulerKey);
- return ps == null ? 0 : ps.getOutstandingAsksCount(resourceName);
+ return ap == null ? 0 : ap.getOutstandingAsksCount(resourceName);
} finally {
readLock.unlock();
}
@@ -617,13 +614,13 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
try {
readLock.lock();
for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
- SchedulingPlacementSet ps = getSchedulingPlacementSet(schedulerKey);
- if (ps != null &&
- ps.getOutstandingAsksCount(ResourceRequest.ANY) > 0) {
+ AppPlacementAllocator ap = getAppPlacementAllocator(schedulerKey);
+ if (ap != null &&
+ ap.getOutstandingAsksCount(ResourceRequest.ANY) > 0) {
LOG.debug("showRequests:" + " application=" + getApplicationId()
+ " headRoom=" + getHeadroom() + " currentConsumption="
+ attemptResourceUsage.getUsed().getMemorySize());
- ps.showRequests();
+ ap.showRequests();
}
}
} finally {
@@ -1334,14 +1331,14 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
this.isAttemptRecovering = isRecovering;
}
- public <N extends SchedulerNode> SchedulingPlacementSet<N> getSchedulingPlacementSet(
+ public <N extends SchedulerNode> AppPlacementAllocator<N> getAppPlacementAllocator(
SchedulerRequestKey schedulerRequestKey) {
- return appSchedulingInfo.getSchedulingPlacementSet(schedulerRequestKey);
+ return appSchedulingInfo.getAppPlacementAllocator(schedulerRequestKey);
}
public Map<String, ResourceRequest> getResourceRequests(
SchedulerRequestKey schedulerRequestKey) {
- return appSchedulingInfo.getSchedulingPlacementSet(schedulerRequestKey)
+ return appSchedulingInfo.getAppPlacementAllocator(schedulerRequestKey)
.getResourceRequests();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
index 12aff02..0c351b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/activities/ActivitiesLogger.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
/**
* Utility for logging scheduler activities
*/
-// FIXME: make sure PlacementSet works with this class
+// FIXME: make sure CandidateNodeSet works with this class
public class ActivitiesLogger {
private static final Log LOG = LogFactory.getLog(ActivitiesLogger.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 250f4e6..183cb36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -876,7 +876,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public CSAssignment assignContainers(Resource clusterResource,
FiCaSchedulerNode node, ResourceLimits resourceLimits,
SchedulingMode schedulingMode) {
- return assignContainers(clusterResource, new SimplePlacementSet<>(node),
+ return assignContainers(clusterResource, new SimpleCandidateNodeSet<>(node),
resourceLimits, schedulingMode);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 3a17d1b..43e7f53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.security.PrivilegedEntity;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -46,12 +45,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
/**
* <code>CSQueue</code> represents a node in the tree of
@@ -188,15 +185,16 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
/**
* Assign containers to applications in the queue or it's children (if any).
* @param clusterResource the resource of the cluster.
- * @param ps {@link PlacementSet} of nodes which resources are available
+ * @param candidates {@link CandidateNodeSet} the nodes that are considered
+ * for the current placement.
* @param resourceLimits how much overall resource of this queue can use.
* @param schedulingMode Type of exclusive check when assign container on a
* NodeManager, see {@link SchedulingMode}.
* @return the assignment
*/
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, ResourceLimits resourceLimits,
- SchedulingMode schedulingMode);
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ ResourceLimits resourceLimits, SchedulingMode schedulingMode);
/**
* A container assigned to the queue has completed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 5e172b8..6f630f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -132,9 +132,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.Lock;
@@ -1183,7 +1183,7 @@ public class CapacityScheduler extends
/**
* We need to make sure when doing allocation, Node should be existed
- * And we will construct a {@link PlacementSet} before proceeding
+ * And we will construct a {@link CandidateNodeSet} before proceeding
*/
private void allocateContainersToNode(NodeId nodeId,
boolean withNodeHeartbeat) {
@@ -1192,8 +1192,10 @@ public class CapacityScheduler extends
int offswitchCount = 0;
int assignedContainers = 0;
- PlacementSet<FiCaSchedulerNode> ps = new SimplePlacementSet<>(node);
- CSAssignment assignment = allocateContainersToNode(ps, withNodeHeartbeat);
+ CandidateNodeSet<FiCaSchedulerNode> candidates =
+ new SimpleCandidateNodeSet<>(node);
+ CSAssignment assignment = allocateContainersToNode(candidates,
+ withNodeHeartbeat);
// Only check if we can allocate more container on the same node when
// scheduling is triggered by node heartbeat
if (null != assignment && withNodeHeartbeat) {
@@ -1210,7 +1212,7 @@ public class CapacityScheduler extends
assignedContainers)) {
// Try to see if it is possible to allocate multiple container for
// the same node heartbeat
- assignment = allocateContainersToNode(ps, true);
+ assignment = allocateContainersToNode(candidates, true);
if (null != assignment
&& assignment.getType() == NodeType.OFF_SWITCH) {
@@ -1237,8 +1239,9 @@ public class CapacityScheduler extends
/*
* Logics of allocate container on a single node (Old behavior)
*/
- private CSAssignment allocateContainerOnSingleNode(PlacementSet<FiCaSchedulerNode> ps,
- FiCaSchedulerNode node, boolean withNodeHeartbeat) {
+ private CSAssignment allocateContainerOnSingleNode(
+ CandidateNodeSet<FiCaSchedulerNode> candidates, FiCaSchedulerNode node,
+ boolean withNodeHeartbeat) {
// Backward compatible way to make sure previous behavior which allocation
// driven by node heartbeat works.
if (getNode(node.getNodeID()) != node) {
@@ -1262,7 +1265,7 @@ public class CapacityScheduler extends
.getApplicationId() + " on node: " + node.getNodeID());
LeafQueue queue = ((LeafQueue) reservedApplication.getQueue());
- assignment = queue.assignContainers(getClusterResource(), ps,
+ assignment = queue.assignContainers(getClusterResource(), candidates,
// TODO, now we only consider limits for parent for non-labeled
// resources, should consider labeled resources as well.
new ResourceLimits(labelManager
@@ -1329,14 +1332,16 @@ public class CapacityScheduler extends
+ node.getUnallocatedResource());
}
- return allocateOrReserveNewContainers(ps, withNodeHeartbeat);
+ return allocateOrReserveNewContainers(candidates, withNodeHeartbeat);
}
private CSAssignment allocateOrReserveNewContainers(
- PlacementSet<FiCaSchedulerNode> ps, boolean withNodeHeartbeat) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ boolean withNodeHeartbeat) {
CSAssignment assignment = getRootQueue().assignContainers(
- getClusterResource(), ps, new ResourceLimits(labelManager
- .getResourceByLabel(ps.getPartition(), getClusterResource())),
+ getClusterResource(), candidates, new ResourceLimits(labelManager
+ .getResourceByLabel(candidates.getPartition(),
+ getClusterResource())),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
assignment.setSchedulingMode(SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
@@ -1346,30 +1351,34 @@ public class CapacityScheduler extends
assignment.getResource(), Resources.none())) {
if (withNodeHeartbeat) {
updateSchedulerHealth(lastNodeUpdateTime,
- PlacementSetUtils.getSingleNode(ps).getNodeID(), assignment);
+ CandidateNodeSetUtils.getSingleNode(candidates).getNodeID(),
+ assignment);
}
return assignment;
}
// Only do non-exclusive allocation when node has node-labels.
- if (StringUtils.equals(ps.getPartition(), RMNodeLabelsManager.NO_LABEL)) {
+ if (StringUtils.equals(candidates.getPartition(),
+ RMNodeLabelsManager.NO_LABEL)) {
return null;
}
// Only do non-exclusive allocation when the node-label supports that
try {
if (rmContext.getNodeLabelManager().isExclusiveNodeLabel(
- ps.getPartition())) {
+ candidates.getPartition())) {
return null;
}
} catch (IOException e) {
- LOG.warn("Exception when trying to get exclusivity of node label=" + ps
+ LOG.warn(
+ "Exception when trying to get exclusivity of node label=" + candidates
.getPartition(), e);
return null;
}
// Try to use NON_EXCLUSIVE
- assignment = getRootQueue().assignContainers(getClusterResource(), ps,
+ assignment = getRootQueue().assignContainers(getClusterResource(),
+ candidates,
// TODO, now we only consider limits for parent for non-labeled
// resources, should consider labeled resources as well.
new ResourceLimits(labelManager
@@ -1386,13 +1395,14 @@ public class CapacityScheduler extends
* New behavior, allocate containers considering multiple nodes
*/
private CSAssignment allocateContainersOnMultiNodes(
- PlacementSet<FiCaSchedulerNode> ps) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates) {
// When this time look at multiple nodes, try schedule if the
// partition has any available resource or killable resource
if (getRootQueue().getQueueCapacities().getUsedCapacity(
- ps.getPartition()) >= 1.0f && preemptionManager.getKillableResource(
- CapacitySchedulerConfiguration.ROOT, ps.getPartition()) == Resources
- .none()) {
+ candidates.getPartition()) >= 1.0f
+ && preemptionManager.getKillableResource(
+ CapacitySchedulerConfiguration.ROOT, candidates.getPartition())
+ == Resources.none()) {
if (LOG.isDebugEnabled()) {
LOG.debug("This node or this node partition doesn't have available or"
+ "killable resource");
@@ -1400,11 +1410,12 @@ public class CapacityScheduler extends
return null;
}
- return allocateOrReserveNewContainers(ps, false);
+ return allocateOrReserveNewContainers(candidates, false);
}
@VisibleForTesting
- CSAssignment allocateContainersToNode(PlacementSet<FiCaSchedulerNode> ps,
+ CSAssignment allocateContainersToNode(
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
boolean withNodeHeartbeat) {
if (rmContext.isWorkPreservingRecoveryEnabled() && !rmContext
.isSchedulerReadyForAllocatingContainers()) {
@@ -1413,14 +1424,14 @@ public class CapacityScheduler extends
// Backward compatible way to make sure previous behavior which allocation
// driven by node heartbeat works.
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
// We have two different logics to handle allocation on single node / multi
// nodes.
if (null != node) {
- return allocateContainerOnSingleNode(ps, node, withNodeHeartbeat);
- } else {
- return allocateContainersOnMultiNodes(ps);
+ return allocateContainerOnSingleNode(candidates, node, withNodeHeartbeat);
+ } else{
+ return allocateContainersOnMultiNodes(candidates);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f2f1baf..e8342d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -69,8 +69,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.utils.Lock;
@@ -970,10 +970,10 @@ public class LeafQueue extends AbstractCSQueue {
limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
}
- private CSAssignment allocateFromReservedContainer(
- Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps,
+ private CSAssignment allocateFromReservedContainer(Resource clusterResource,
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
if (null == node) {
return null;
}
@@ -987,7 +987,8 @@ public class LeafQueue extends AbstractCSQueue {
ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
node.getNodeID(), SystemClock.getInstance().getTime(), application);
CSAssignment assignment = application.assignContainers(clusterResource,
- ps, currentResourceLimits, schedulingMode, reservedContainer);
+ candidates, currentResourceLimits, schedulingMode,
+ reservedContainer);
return assignment;
}
}
@@ -997,43 +998,44 @@ public class LeafQueue extends AbstractCSQueue {
@Override
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, ResourceLimits currentResourceLimits,
- SchedulingMode schedulingMode) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ ResourceLimits currentResourceLimits, SchedulingMode schedulingMode) {
updateCurrentResourceLimits(currentResourceLimits, clusterResource);
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
if (LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: partition=" + ps.getPartition()
+ LOG.debug("assignContainers: partition=" + candidates.getPartition()
+ " #applications=" + orderingPolicy.getNumSchedulableEntities());
}
- setPreemptionAllowed(currentResourceLimits, ps.getPartition());
+ setPreemptionAllowed(currentResourceLimits, candidates.getPartition());
// Check for reserved resources, try to allocate reserved container first.
CSAssignment assignment = allocateFromReservedContainer(clusterResource,
- ps, currentResourceLimits, schedulingMode);
+ candidates, currentResourceLimits, schedulingMode);
if (null != assignment) {
return assignment;
}
// if our queue cannot access this node, just return
if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
- && !accessibleToPartition(ps.getPartition())) {
+ && !accessibleToPartition(candidates.getPartition())) {
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
getParent().getQueueName(), getQueueName(), ActivityState.REJECTED,
- ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + ps
+ ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + candidates
.getPartition());
return CSAssignment.NULL_ASSIGNMENT;
}
// Check if this queue need more resource, simply skip allocation if this
// queue doesn't need more resources.
- if (!hasPendingResourceRequest(ps.getPartition(), clusterResource,
+ if (!hasPendingResourceRequest(candidates.getPartition(), clusterResource,
schedulingMode)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip this queue=" + getQueuePath()
+ ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-partition=" + ps.getPartition());
+ + schedulingMode.name() + " node-partition=" + candidates
+ .getPartition());
}
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
@@ -1078,7 +1080,8 @@ public class LeafQueue extends AbstractCSQueue {
cachedUserLimit = cul.userLimit;
}
Resource userLimit = computeUserLimitAndSetHeadroom(application,
- clusterResource, ps.getPartition(), schedulingMode, cachedUserLimit);
+ clusterResource, candidates.getPartition(), schedulingMode,
+ cachedUserLimit);
if (cul == null) {
cul = new CachedUserLimit(userLimit);
userLimits.put(application.getUser(), cul);
@@ -1106,7 +1109,7 @@ public class LeafQueue extends AbstractCSQueue {
// Try to schedule
assignment = application.assignContainers(clusterResource,
- ps, currentResourceLimits, schedulingMode, null);
+ candidates, currentResourceLimits, schedulingMode, null);
if (LOG.isDebugEnabled()) {
LOG.debug("post-assignContainers for application " + application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 2c288f2..d61951b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -65,8 +65,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -479,16 +479,16 @@ public class ParentQueue extends AbstractCSQueue {
@Override
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, ResourceLimits resourceLimits,
- SchedulingMode schedulingMode) {
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ ResourceLimits resourceLimits, SchedulingMode schedulingMode) {
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
// if our queue cannot access this node, just return
if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
- && !accessibleToPartition(ps.getPartition())) {
+ && !accessibleToPartition(candidates.getPartition())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it is not able to access partition=" + ps
+ + ", because it is not able to access partition=" + candidates
.getPartition());
}
@@ -506,12 +506,12 @@ public class ParentQueue extends AbstractCSQueue {
// Check if this queue need more resource, simply skip allocation if this
// queue doesn't need more resources.
- if (!super.hasPendingResourceRequest(ps.getPartition(), clusterResource,
- schedulingMode)) {
+ if (!super.hasPendingResourceRequest(candidates.getPartition(),
+ clusterResource, schedulingMode)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip this queue=" + getQueuePath()
+ ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-partition=" + ps
+ + schedulingMode.name() + " node-partition=" + candidates
.getPartition());
}
@@ -538,7 +538,8 @@ public class ParentQueue extends AbstractCSQueue {
// Are we over maximum-capacity for this queue?
// This will also consider parent's limits and also continuous reservation
// looking
- if (!super.canAssignToThisQueue(clusterResource, ps.getPartition(),
+ if (!super.canAssignToThisQueue(clusterResource,
+ candidates.getPartition(),
resourceLimits, Resources
.createResource(getMetrics().getReservedMB(),
getMetrics().getReservedVirtualCores()), schedulingMode)) {
@@ -556,7 +557,7 @@ public class ParentQueue extends AbstractCSQueue {
// Schedule
CSAssignment assignedToChild = assignContainersToChildQueues(
- clusterResource, ps, resourceLimits, schedulingMode);
+ clusterResource, candidates, resourceLimits, schedulingMode);
assignment.setType(assignedToChild.getType());
assignment.setRequestLocalityType(
assignedToChild.getRequestLocalityType());
@@ -710,7 +711,7 @@ public class ParentQueue extends AbstractCSQueue {
}
private CSAssignment assignContainersToChildQueues(Resource cluster,
- PlacementSet<FiCaSchedulerNode> ps, ResourceLimits limits,
+ CandidateNodeSet<FiCaSchedulerNode> candidates, ResourceLimits limits,
SchedulingMode schedulingMode) {
CSAssignment assignment = CSAssignment.NULL_ASSIGNMENT;
@@ -719,7 +720,7 @@ public class ParentQueue extends AbstractCSQueue {
// Try to assign to most 'under-served' sub-queue
for (Iterator<CSQueue> iter = sortAndGetChildrenAllocationIterator(
- ps.getPartition()); iter.hasNext(); ) {
+ candidates.getPartition()); iter.hasNext(); ) {
CSQueue childQueue = iter.next();
if(LOG.isDebugEnabled()) {
LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
@@ -729,10 +730,10 @@ public class ParentQueue extends AbstractCSQueue {
// Get ResourceLimits of child queue before assign containers
ResourceLimits childLimits =
getResourceLimitsOfChild(childQueue, cluster, parentLimits,
- ps.getPartition());
-
- CSAssignment childAssignment = childQueue.assignContainers(cluster, ps,
- childLimits, schedulingMode);
+ candidates.getPartition());
+
+ CSAssignment childAssignment = childQueue.assignContainers(cluster,
+ candidates, childLimits, schedulingMode);
if(LOG.isDebugEnabled()) {
LOG.debug("Assigned to queue: " + childQueue.getQueuePath() +
" stats: " + childQueue + " --> " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
index 5809d86..95e0533 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocat
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -34,7 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssign
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -176,13 +175,14 @@ public abstract class AbstractContainerAllocator {
* </ul>
*
* @param clusterResource clusterResource
- * @param ps PlacementSet
+ * @param candidates CandidateNodeSet
* @param schedulingMode scheduling mode (exclusive or nonexclusive)
* @param resourceLimits resourceLimits
* @param reservedContainer reservedContainer
* @return CSAssignemnt proposal
*/
public abstract CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, RMContainer reservedContainer);
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ SchedulingMode schedulingMode, ResourceLimits resourceLimits,
+ RMContainer reservedContainer);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
index 4879fae..9df03b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/ContainerAllocator.java
@@ -21,16 +21,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocat
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerAllocator extends AbstractContainerAllocator {
private AbstractContainerAllocator regularContainerAllocator;
@@ -50,10 +48,11 @@ public class ContainerAllocator extends AbstractContainerAllocator {
@Override
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, RMContainer reservedContainer) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ SchedulingMode schedulingMode, ResourceLimits resourceLimits,
+ RMContainer reservedContainer) {
return regularContainerAllocator.assignContainers(clusterResource,
- ps, schedulingMode, resourceLimits, reservedContainer);
+ candidates, schedulingMode, resourceLimits, reservedContainer);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 72dfbdd..69e90c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
@@ -50,9 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssign
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -91,15 +91,16 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
/*
* Pre-check if we can allocate a pending resource request
- * (given schedulerKey) to a given PlacementSet.
+ * (given schedulerKey) to a given CandidateNodeSet.
* We will consider stuffs like exclusivity, pending resource, node partition,
* headroom, etc.
*/
- private ContainerAllocation preCheckForPlacementSet(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey) {
+ private ContainerAllocation preCheckForNodeCandidateSet(
+ Resource clusterResource, CandidateNodeSet<FiCaSchedulerNode> candidates,
+ SchedulingMode schedulingMode, ResourceLimits resourceLimits,
+ SchedulerRequestKey schedulerKey) {
Priority priority = schedulerKey.getPriority();
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
PendingAsk offswitchPendingAsk = application.getPendingAsk(schedulerKey,
ResourceRequest.ANY);
@@ -164,7 +165,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
if (!checkHeadroom(clusterResource, resourceLimits, required,
- ps.getPartition())) {
+ candidates.getPartition())) {
if (LOG.isDebugEnabled()) {
LOG.debug("cannot allocate required resource=" + required
+ " because of headroom");
@@ -182,7 +183,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// Only do this when request associated with given scheduler key accepts
// NO_LABEL under RESPECT_EXCLUSIVITY mode
if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL,
- appInfo.getSchedulingPlacementSet(schedulerKey)
+ appInfo.getAppPlacementAllocator(schedulerKey)
.getPrimaryRequestedNodePartition())) {
missedNonPartitionedRequestSchedulingOpportunity =
application.addMissedNonPartitionedRequestSchedulingOpportunity(
@@ -265,7 +266,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
SchedulerRequestKey schedulerKey, int clusterNodes) {
// Estimate: Required unique resources (i.e. hosts + racks)
int requiredResources = Math.max(
- application.getSchedulingPlacementSet(schedulerKey)
+ application.getAppPlacementAllocator(schedulerKey)
.getUniqueLocationAsks() - 1, 0);
// waitFactor can't be more than '1'
@@ -780,15 +781,15 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
private ContainerAllocation allocate(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey,
- RMContainer reservedContainer) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ SchedulingMode schedulingMode, ResourceLimits resourceLimits,
+ SchedulerRequestKey schedulerKey, RMContainer reservedContainer) {
// Do checks before determining which node to allocate
// Directly return if this check fails.
ContainerAllocation result;
if (reservedContainer == null) {
- result = preCheckForPlacementSet(clusterResource, ps, schedulingMode,
- resourceLimits, schedulerKey);
+ result = preCheckForNodeCandidateSet(clusterResource, candidates,
+ schedulingMode, resourceLimits, schedulerKey);
if (null != result) {
return result;
}
@@ -801,14 +802,14 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
}
}
- SchedulingPlacementSet<FiCaSchedulerNode> schedulingPS =
- application.getAppSchedulingInfo().getSchedulingPlacementSet(
+ AppPlacementAllocator<FiCaSchedulerNode> schedulingPS =
+ application.getAppSchedulingInfo().getAppPlacementAllocator(
schedulerKey);
result = ContainerAllocation.PRIORITY_SKIPPED;
Iterator<FiCaSchedulerNode> iter = schedulingPS.getPreferredNodeIterator(
- ps);
+ candidates);
while (iter.hasNext()) {
FiCaSchedulerNode node = iter.next();
@@ -827,19 +828,20 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
@Override
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, SchedulingMode schedulingMode,
- ResourceLimits resourceLimits,
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ SchedulingMode schedulingMode, ResourceLimits resourceLimits,
RMContainer reservedContainer) {
- FiCaSchedulerNode node = PlacementSetUtils.getSingleNode(ps);
+ FiCaSchedulerNode node = CandidateNodeSetUtils.getSingleNode(candidates);
if (reservedContainer == null) {
// Check if application needs more resource, skip if it doesn't need more.
if (!application.hasPendingResourceRequest(rc,
- ps.getPartition(), clusterResource, schedulingMode)) {
+ candidates.getPartition(), clusterResource, schedulingMode)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip app_attempt=" + application.getApplicationAttemptId()
+ ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-label=" + ps.getPartition());
+ + schedulingMode.name() + " node-label=" + candidates
+ .getPartition());
}
ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
activitiesManager, node, application, application.getPriority(),
@@ -849,9 +851,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// Schedule in priority order
for (SchedulerRequestKey schedulerKey : application.getSchedulerKeys()) {
- ContainerAllocation result =
- allocate(clusterResource, ps, schedulingMode, resourceLimits,
- schedulerKey, null);
+ ContainerAllocation result = allocate(clusterResource, candidates,
+ schedulingMode, resourceLimits, schedulerKey, null);
AllocationState allocationState = result.getAllocationState();
if (allocationState == AllocationState.PRIORITY_SKIPPED) {
@@ -869,7 +870,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
return CSAssignment.SKIP_ASSIGNMENT;
} else {
ContainerAllocation result =
- allocate(clusterResource, ps, schedulingMode, resourceLimits,
+ allocate(clusterResource, candidates, schedulingMode, resourceLimits,
reservedContainer.getReservedSchedulerKey(), reservedContainer);
return getCSAssignmentFromAllocateResult(clusterResource, result,
reservedContainer, node);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index a12c5ec..40405fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -76,8 +76,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerA
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.AppPlacementAllocator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -224,10 +224,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return null;
}
- SchedulingPlacementSet<FiCaSchedulerNode> ps =
- appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
+ AppPlacementAllocator<FiCaSchedulerNode> ps =
+ appSchedulingInfo.getAppPlacementAllocator(schedulerKey);
if (null == ps) {
- LOG.warn("Failed to get " + SchedulingPlacementSet.class.getName()
+ LOG.warn("Failed to get " + AppPlacementAllocator.class.getName()
+ " for application=" + getApplicationId() + " schedulerRequestKey="
+ schedulerKey);
return null;
@@ -636,8 +636,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
Map<String, Resource> ret = new HashMap<>();
for (SchedulerRequestKey schedulerKey : appSchedulingInfo
.getSchedulerKeys()) {
- SchedulingPlacementSet<FiCaSchedulerNode> ps =
- appSchedulingInfo.getSchedulingPlacementSet(schedulerKey);
+ AppPlacementAllocator<FiCaSchedulerNode> ps =
+ appSchedulingInfo.getAppPlacementAllocator(schedulerKey);
String nodePartition = ps.getPrimaryRequestedNodePartition();
Resource res = ret.get(nodePartition);
@@ -844,8 +844,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
public CSAssignment assignContainers(Resource clusterResource,
- PlacementSet<FiCaSchedulerNode> ps, ResourceLimits currentResourceLimits,
- SchedulingMode schedulingMode, RMContainer reservedContainer) {
+ CandidateNodeSet<FiCaSchedulerNode> ps,
+ ResourceLimits currentResourceLimits, SchedulingMode schedulingMode,
+ RMContainer reservedContainer) {
if (LOG.isDebugEnabled()) {
LOG.debug("pre-assignContainers for application "
+ getApplicationId());
@@ -962,9 +963,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
@Override
@SuppressWarnings("unchecked")
- public SchedulingPlacementSet<FiCaSchedulerNode> getSchedulingPlacementSet(
+ public AppPlacementAllocator<FiCaSchedulerNode> getAppPlacementAllocator(
SchedulerRequestKey schedulerRequestKey) {
- return super.getSchedulingPlacementSet(schedulerRequestKey);
+ return super.getAppPlacementAllocator(schedulerRequestKey);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 157d264..bbd4418 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -1019,7 +1019,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
if (offswitchAsk.getCount() > 0) {
- if (getSchedulingPlacementSet(schedulerKey).getUniqueLocationAsks()
+ if (getAppPlacementAllocator(schedulerKey).getUniqueLocationAsks()
<= 1 || allowedLocality.equals(NodeType.OFF_SWITCH)) {
if (LOG.isTraceEnabled()) {
LOG.trace("Assign container on " + node.getNodeName()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
new file mode 100644
index 0000000..63b22a3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * This class has the following functionality:
+ * 1) Keeps track of pending resource requests when following events happen:
+ * - New ResourceRequests are added to scheduler.
+ * - New containers get allocated.
+ *
+ * 2) Determines the order that the nodes given in the {@link CandidateNodeSet}
+ * will be used for allocating containers.
+ * </p>
+ *
+ * <p>
+ * And different set of resource requests (E.g., resource requests with the
+ * same schedulerKey) can have one instance of AppPlacementAllocator, each
+ * AppPlacementAllocator can have different ways to order nodes depends on
+ * requests.
+ * </p>
+ */
+public interface AppPlacementAllocator<N extends SchedulerNode> {
+ /**
+ * Get iterator of preferred node depends on requirement and/or availability
+ * @param candidateNodeSet input CandidateNodeSet
+ * @return iterator of preferred node
+ */
+ Iterator<N> getPreferredNodeIterator(CandidateNodeSet<N> candidateNodeSet);
+
+ /**
+ * Replace existing ResourceRequest by the new requests
+ *
+ * @param requests new ResourceRequests
+ * @param recoverPreemptedRequestForAContainer if we're recovering resource
+ * requests for preempted container
+ * @return true if total pending resource changed
+ */
+ ResourceRequestUpdateResult updateResourceRequests(
+ Collection<ResourceRequest> requests,
+ boolean recoverPreemptedRequestForAContainer);
+
+ /**
+ * Get pending ResourceRequests by given schedulerRequestKey
+ * @return Map of resourceName to ResourceRequest
+ */
+ Map<String, ResourceRequest> getResourceRequests();
+
+ /**
+ * Get pending ask for given resourceName. If there's no such pendingAsk,
+ * returns {@link PendingAsk#ZERO}
+ *
+ * @param resourceName resourceName
+ * @return PendingAsk
+ */
+ PendingAsk getPendingAsk(String resourceName);
+
+ /**
+ * Get #pending-allocations for given resourceName. If there's no such
+ * pendingAsk, returns 0
+ *
+ * @param resourceName resourceName
+ * @return #pending-allocations
+ */
+ int getOutstandingAsksCount(String resourceName);
+
+ /**
+ * Notify container allocated.
+ * @param schedulerKey SchedulerRequestKey for this ResourceRequest
+ * @param type Type of the allocation
+ * @param node Which node this container allocated on
+ * @return list of ResourceRequests deducted
+ */
+ List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
+ NodeType type, SchedulerNode node);
+
+ /**
+ * Returns list of accepted resourceNames.
+ * @return Iterator of accepted resourceNames
+ */
+ Iterator<String> getAcceptedResouceNames();
+
+ /**
+ * We can still have pending requirement for a given NodeType and node
+ * @param type Locality Type
+ * @param node which node we will allocate on
+ * @return true if we has pending requirement
+ */
+ boolean canAllocate(NodeType type, SchedulerNode node);
+
+ /**
+ * Can delay to give locality?
+ * TODO: This should be moved out of AppPlacementAllocator
+ * and should belong to specific delay scheduling policy impl.
+ * See YARN-7457 for more details.
+ *
+ * @param resourceName resourceName
+ * @return can/cannot
+ */
+ boolean canDelayTo(String resourceName);
+
+ /**
+ * Does this {@link AppPlacementAllocator} accept resources on nodePartition?
+ *
+ * @param nodePartition nodePartition
+ * @param schedulingMode schedulingMode
+ * @return accepted/not
+ */
+ boolean acceptNodePartition(String nodePartition,
+ SchedulingMode schedulingMode);
+
+ /**
+ * It is possible that one request can accept multiple node partition,
+ * So this method returns primary node partition for pending resource /
+ * headroom calculation.
+ *
+ * @return primary requested node partition
+ */
+ String getPrimaryRequestedNodePartition();
+
+ /**
+ * @return number of unique location asks with #pending greater than 0,
+ * (like /rack1, host1, etc.).
+ *
+ * TODO: This should be moved out of AppPlacementAllocator
+ * and should belong to specific delay scheduling policy impl.
+ * See YARN-7457 for more details.
+ */
+ int getUniqueLocationAsks();
+
+ /**
+ * Print human-readable requests to LOG debug.
+ */
+ void showRequests();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSet.java
new file mode 100644
index 0000000..6f127c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSet.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+import java.util.Map;
+
+/**
+ * A group of nodes which can be allocated by scheduler.
+ *
+ * It will have following part:
+ *
+ * 1) A map of nodes which can be schedule-able.
+ * 2) Version of the node set, version should be updated if any node added or
+ * removed from the node set. This will be used by
+ * {@link AppPlacementAllocator} or other class to check if it's required to
+ * invalidate local caches, etc.
+ * 3) Node partition of the candidate set.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface CandidateNodeSet<N extends SchedulerNode> {
+ /**
+ * Get all nodes for this CandidateNodeSet
+ * @return all nodes for this CandidateNodeSet
+ */
+ Map<NodeId, N> getAllNodes();
+
+ /**
+ * Version of the CandidateNodeSet, can help {@link AppPlacementAllocator} to
+ * decide if update is required
+ * @return version
+ */
+ long getVersion();
+
+ /**
+ * Node partition of the node set.
+ * @return node partition
+ */
+ String getPartition();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSetUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSetUtils.java
new file mode 100644
index 0000000..157d2ee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/CandidateNodeSetUtils.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+/**
+ * Utility methods for {@link CandidateNodeSet}.
+ */
+public final class CandidateNodeSetUtils {
+
+ private CandidateNodeSetUtils() {
+ }
+
+ /*
+ * If the {@link CandidateNodeSet} only has one entry, return it. Otherwise,
+ * return null.
+ */
+ public static <N extends SchedulerNode> N getSingleNode(
+ CandidateNodeSet<N> candidates) {
+ N node = null;
+ if (1 == candidates.getAllNodes().size()) {
+ node = candidates.getAllNodes().values().iterator().next();
+ }
+
+ return node;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index e3c108a..b9d5b82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableTable;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.junit.Assert;
@@ -52,6 +53,8 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
when(q.getQueueCapacities()).thenReturn(qc);
when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i]));
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
list.add(q);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1108f1a..0132348 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -354,10 +354,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 18;
+ int numExpectedElements = 20;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 31;
+ numExpectedElements = 33;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-6909. Use
LightWeightedResource when number of resource types more than two. (Sunil G
via wangda)
Posted by su...@apache.org.
YARN-6909. Use LightWeightedResource when number of resource types more than two. (Sunil G via wangda)
Change-Id: I90e021c5dea7abd9ec6bd73b2287c8adebe14595
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd07038f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd07038f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd07038f
Branch: refs/heads/YARN-5881
Commit: dd07038ffae33a5e73eb331477d43eaf3f4c2aaa
Parents: 1883a00
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 9 14:51:15 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Nov 9 14:51:15 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Resource.java | 48 ++++++----
.../api/records/impl/LightWeightResource.java | 94 +++++++++++++-------
.../api/records/impl/pb/ResourcePBImpl.java | 88 +++++++++---------
.../scheduler/ClusterNodeTracker.java | 2 +-
4 files changed, 141 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd07038f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index be292ff..65b5dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -28,9 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
@@ -75,34 +75,27 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- Resource ret = Records.newRecord(Resource.class);
- ret.setMemorySize(memory);
- ret.setVirtualCores(vCores);
- return ret;
- }
return new LightWeightResource(memory, vCores);
}
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- Resource ret = Records.newRecord(Resource.class);
- ret.setMemorySize(memory);
- ret.setVirtualCores(vCores);
- return ret;
- }
return new LightWeightResource(memory, vCores);
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static Resource newInstance(Resource resource) {
- Resource ret = Resource.newInstance(resource.getMemorySize(),
- resource.getVirtualCores());
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- Resource.copy(resource, ret);
+ Resource ret;
+ int numberOfKnownResourceTypes = ResourceUtils
+ .getNumberOfKnownResourceTypes();
+ if (numberOfKnownResourceTypes > 2) {
+ ret = new LightWeightResource(resource.getMemorySize(),
+ resource.getVirtualCores(), resource.getResources());
+ } else {
+ ret = new LightWeightResource(resource.getMemorySize(),
+ resource.getVirtualCores());
}
return ret;
}
@@ -411,7 +404,7 @@ public abstract class Resource implements Comparable<Resource> {
int arrLenOther = otherResources.length;
// compare memory and vcores first(in that order) to preserve
- // existing behaviour
+ // existing behavior.
for (int i = 0; i < arrLenThis; i++) {
ResourceInformation otherEntry;
try {
@@ -483,4 +476,23 @@ public abstract class Resource implements Comparable<Resource> {
}
return Long.valueOf(value).intValue();
}
+
+ /**
+ * Create ResourceInformation with basic fields.
+ * @param name Resource Type Name
+ * @param unit Default unit of provided resource type
+ * @param value Value associated with giveb resource
+ * @return ResourceInformation object
+ */
+ protected static ResourceInformation newDefaultInformation(String name,
+ String unit, long value) {
+ ResourceInformation ri = new ResourceInformation();
+ ri.setName(name);
+ ri.setValue(value);
+ ri.setResourceType(ResourceTypes.COUNTABLE);
+ ri.setUnitsWithoutValidation(unit);
+ ri.setMinimumAllocation(0);
+ ri.setMaximumAllocation(Long.MAX_VALUE);
+ return ri;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd07038f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
index 7b07bbd..a6e6432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
@@ -23,14 +23,13 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_MB;
-import static org.apache.hadoop.yarn.api.records.ResourceInformation.MEMORY_URI;
-import static org.apache.hadoop.yarn.api.records.ResourceInformation.VCORES_URI;
+import static org.apache.hadoop.yarn.api.records.ResourceInformation.*;
/**
* <p>
- * <code>LightResource</code> extends Resource to handle base resources such
+ * <code>LightWeightResource</code> extends Resource to handle base resources such
* as memory and CPU.
* TODO: We have a long term plan to use AbstractResource when additional
* resource types are to be handled as well.
@@ -66,27 +65,42 @@ public class LightWeightResource extends Resource {
private ResourceInformation memoryResInfo;
private ResourceInformation vcoresResInfo;
- public LightWeightResource(long memory, long vcores) {
- this.memoryResInfo = LightWeightResource.newDefaultInformation(MEMORY_URI,
- MEMORY_MB.getUnits(), memory);
- this.vcoresResInfo = LightWeightResource.newDefaultInformation(VCORES_URI,
- "", vcores);
+ public LightWeightResource(long memory, int vcores) {
+ int numberOfKnownResourceTypes = ResourceUtils
+ .getNumberOfKnownResourceTypes();
+ initResourceInformations(memory, vcores, numberOfKnownResourceTypes);
+
+ if (numberOfKnownResourceTypes > 2) {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
+ for (int i = 2; i < numberOfKnownResourceTypes; i++) {
+ resources[i] = new ResourceInformation();
+ ResourceInformation.copy(types[i], resources[i]);
+ }
+ }
+ }
- resources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
- resources[MEMORY_INDEX] = memoryResInfo;
- resources[VCORES_INDEX] = vcoresResInfo;
+ public LightWeightResource(long memory, int vcores,
+ ResourceInformation[] source) {
+ int numberOfKnownResourceTypes = ResourceUtils
+ .getNumberOfKnownResourceTypes();
+ initResourceInformations(memory, vcores, numberOfKnownResourceTypes);
+
+ for (int i = 2; i < numberOfKnownResourceTypes; i++) {
+ resources[i] = new ResourceInformation();
+ ResourceInformation.copy(source[i], resources[i]);
+ }
}
- private static ResourceInformation newDefaultInformation(String name,
- String unit, long value) {
- ResourceInformation ri = new ResourceInformation();
- ri.setName(name);
- ri.setValue(value);
- ri.setResourceType(ResourceTypes.COUNTABLE);
- ri.setUnitsWithoutValidation(unit);
- ri.setMinimumAllocation(0);
- ri.setMaximumAllocation(Long.MAX_VALUE);
- return ri;
+ private void initResourceInformations(long memory, int vcores,
+ int numberOfKnownResourceTypes) {
+ this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(),
+ memory);
+ this.vcoresResInfo = newDefaultInformation(VCORES_URI, VCORES.getUnits(),
+ vcores);
+
+ resources = new ResourceInformation[numberOfKnownResourceTypes];
+ resources[MEMORY_INDEX] = memoryResInfo;
+ resources[VCORES_INDEX] = vcoresResInfo;
}
@Override
@@ -135,21 +149,41 @@ public class LightWeightResource extends Resource {
return false;
}
+ if (resources.length > 2) {
+ ResourceInformation[] otherVectors = other.getResources();
+
+ if (resources.length != otherVectors.length) {
+ return false;
+ }
+
+ for (int i = 2; i < resources.length; i++) {
+ ResourceInformation a = resources[i];
+ ResourceInformation b = otherVectors[i];
+ if ((a != b) && ((a == null) || !a.equals(b))) {
+ return false;
+ }
+ }
+ }
+
return true;
}
@Override
public int compareTo(Resource other) {
// compare memory and vcores first(in that order) to preserve
- // existing behaviour
- long diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- return this.getVirtualCores() - other.getVirtualCores();
- } else if (diff > 0){
- return 1;
- } else {
- return -1;
+ // existing behavior.
+ if (resources.length <= 2) {
+ long diff = this.getMemorySize() - other.getMemorySize();
+ if (diff == 0) {
+ return this.getVirtualCores() - other.getVirtualCores();
+ } else if (diff > 0) {
+ return 1;
+ } else {
+ return -1;
+ }
}
+
+ return super.compareTo(other);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd07038f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 4ae64c2..401e0c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import java.util.Map;
-
@Private
@Unstable
public class ResourcePBImpl extends Resource {
@@ -50,11 +49,14 @@ public class ResourcePBImpl extends Resource {
static ResourceProto getProto(Resource r) {
final ResourcePBImpl pb;
if (r instanceof ResourcePBImpl) {
- pb = (ResourcePBImpl)r;
+ pb = (ResourcePBImpl) r;
} else {
pb = new ResourcePBImpl();
pb.setMemorySize(r.getMemorySize());
pb.setVirtualCores(r.getVirtualCores());
+ for(ResourceInformation res : r.getResources()) {
+ pb.setResourceInformation(res.getName(), res);
+ }
}
return pb.getProto();
}
@@ -111,7 +113,7 @@ public class ResourcePBImpl extends Resource {
@Override
public void setMemorySize(long memory) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.MEMORY_URI).setValue(memory);
+ resources[MEMORY_INDEX].setValue(memory);
}
@Override
@@ -123,7 +125,7 @@ public class ResourcePBImpl extends Resource {
@Override
public void setVirtualCores(int vCores) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.VCORES_URI).setValue(vCores);
+ resources[VCORES_INDEX].setValue(vCores);
}
private void initResources() {
@@ -131,31 +133,51 @@ public class ResourcePBImpl extends Resource {
return;
}
ResourceProtoOrBuilder p = viaProto ? proto : builder;
- initResourcesMap();
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
Map<String, Integer> indexMap = ResourceUtils.getResourceTypeIndex();
+ resources = new ResourceInformation[types.length];
+
for (ResourceInformationProto entry : p.getResourceValueMapList()) {
- ResourceTypes type =
- entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
- ResourceTypes.COUNTABLE;
-
- // When unit not specified in proto, use the default unit.
- String units =
- entry.hasUnits() ? entry.getUnits() : ResourceUtils.getDefaultUnit(
- entry.getKey());
- long value = entry.hasValue() ? entry.getValue() : 0L;
- ResourceInformation ri = ResourceInformation
- .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
Integer index = indexMap.get(entry.getKey());
if (index == null) {
- LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping");
+ LOG.warn("Got unknown resource type: " + entry.getKey() + "; skipping");
} else {
- resources[index].setResourceType(ri.getResourceType());
- resources[index].setUnits(ri.getUnits());
- resources[index].setValue(value);
+ resources[index] = newDefaultInformation(types[index], entry);
}
}
+
+ resources[MEMORY_INDEX] = ResourceInformation
+ .newInstance(ResourceInformation.MEMORY_MB);
+ resources[VCORES_INDEX] = ResourceInformation
+ .newInstance(ResourceInformation.VCORES);
this.setMemorySize(p.getMemory());
this.setVirtualCores(p.getVirtualCores());
+
+ // Update missing resource information on respective index.
+ updateResourceInformationMap(types);
+ }
+
+ private void updateResourceInformationMap(ResourceInformation[] types) {
+ for (int i = 0; i < types.length; i++) {
+ if (resources[i] == null) {
+ resources[i] = ResourceInformation.newInstance(types[i]);
+ }
+ }
+ }
+
+ private static ResourceInformation newDefaultInformation(
+ ResourceInformation resourceInformation, ResourceInformationProto entry) {
+ ResourceInformation ri = new ResourceInformation();
+ ri.setName(resourceInformation.getName());
+ ri.setMinimumAllocation(resourceInformation.getMinimumAllocation());
+ ri.setMaximumAllocation(resourceInformation.getMaximumAllocation());
+ ri.setResourceType(entry.hasType()
+ ? ProtoUtils.convertFromProtoFormat(entry.getType())
+ : ResourceTypes.COUNTABLE);
+ ri.setUnits(
+ entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits());
+ ri.setValue(entry.hasValue() ? entry.getValue() : 0L);
+ return ri;
}
@Override
@@ -166,10 +188,8 @@ public class ResourcePBImpl extends Resource {
throw new IllegalArgumentException(
"resource and/or resourceInformation cannot be null");
}
- if (!resource.equals(resourceInformation.getName())) {
- resourceInformation.setName(resource);
- }
- ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ ResourceInformation storedResourceInfo = super.getResourceInformation(
+ resource);
ResourceInformation.copy(resourceInformation, storedResourceInfo);
}
@@ -195,25 +215,9 @@ public class ResourcePBImpl extends Resource {
return super.getResourceValue(resource);
}
- private void initResourcesMap() {
- if (resources == null) {
- ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
- if (types == null) {
- throw new YarnRuntimeException(
- "Got null return value from ResourceUtils.getResourceTypes()");
- }
-
- resources = new ResourceInformation[types.length];
- for (ResourceInformation entry : types) {
- int index = ResourceUtils.getResourceTypeIndex().get(entry.getName());
- resources[index] = ResourceInformation.newInstance(entry);
- }
- }
- }
-
synchronized private void mergeLocalToBuilder() {
builder.clearResourceValueMap();
- if(resources != null && resources.length != 0) {
+ if (resources != null && resources.length != 0) {
for (ResourceInformation resInfo : resources) {
ResourceInformationProto.Builder e = ResourceInformationProto
.newBuilder();
@@ -236,4 +240,4 @@ public class ResourcePBImpl extends Resource {
proto = builder.build();
viaProto = true;
}
-}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd07038f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 60ef390..9e54ac6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -55,7 +55,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
private Map<String, N> nodeNameToNodeMap = new HashMap<>();
private Map<String, List<N>> nodesPerRack = new HashMap<>();
- private final Resource clusterCapacity = Resources.clone(Resources.none());
+ private Resource clusterCapacity = Resources.createResource(0, 0);
private volatile Resource staleClusterCapacity =
Resources.clone(Resources.none());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-12797. Add Test for NFS mount
of not supported filesystems like (file:///). Contributed by Mukul Kumar
Singh.
Posted by su...@apache.org.
HDFS-12797. Add Test for NFS mount of not supported filesystems like (file:///). Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a1bd9a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a1bd9a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a1bd9a4
Branch: refs/heads/YARN-5881
Commit: 8a1bd9a4f4b8864aa560094a53d43ef732d378e5
Parents: 3c6adda
Author: Jitendra Pandey <ji...@apache.org>
Authored: Thu Nov 9 23:53:17 2017 -0800
Committer: Jitendra Pandey <ji...@apache.org>
Committed: Thu Nov 9 23:53:17 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/nfs/nfs3/TestExportsTable.java | 88 +++++++++++++++++++-
1 file changed, 87 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a1bd9a4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
index 211a166..a5c3e7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestExportsTable.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.nfs.nfs3;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.nio.file.FileSystemException;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -33,9 +34,14 @@ import org.apache.hadoop.hdfs.nfs.conf.NfsConfigKeys;
import org.apache.hadoop.hdfs.nfs.conf.NfsConfiguration;
import org.apache.hadoop.hdfs.nfs.mount.Mountd;
import org.apache.hadoop.hdfs.nfs.mount.RpcProgramMountd;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
public class TestExportsTable {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
@Test
public void testHdfsExportPoint() throws IOException {
@@ -70,7 +76,7 @@ public class TestExportsTable {
}
@Test
- public void testViewFsExportPoint() throws IOException {
+ public void testViewFsMultipleExportPoint() throws IOException {
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
String clusterName = RandomStringUtils.randomAlphabetic(10);
@@ -183,6 +189,56 @@ public class TestExportsTable {
}
@Test
+ public void testViewFsRootExportPoint() throws IOException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+ String clusterName = RandomStringUtils.randomAlphabetic(10);
+
+ String exportPoint = "/";
+ config.setStrings(NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY, exportPoint);
+ config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_SCHEME + "://" + clusterName);
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set("nfs.http.address", "0.0.0.0:0");
+
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(config).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+ DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+ cluster.waitActive();
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(config, clusterName, "/hdfs1",
+ hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(config, clusterName, "/hdfs2",
+ hdfs2.makeQualified(base2).toUri());
+
+ exception.expect(FileSystemException.class);
+ exception.
+ expectMessage("Only HDFS is supported as underlyingFileSystem, "
+ + "fs scheme:viewfs");
+ // Start nfs
+ final Nfs3 nfsServer = new Nfs3(config);
+ nfsServer.startServiceInternal(false);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
public void testHdfsInternalExportPoint() throws IOException {
NfsConfiguration config = new NfsConfiguration();
MiniDFSCluster cluster = null;
@@ -219,4 +275,34 @@ public class TestExportsTable {
}
}
}
+
+ @Test
+ public void testInvalidFsExport() throws IOException {
+ NfsConfiguration config = new NfsConfiguration();
+ MiniDFSCluster cluster = null;
+
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
+ config.set("nfs.http.address", "0.0.0.0:0");
+
+ try {
+ cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ cluster.waitActive();
+ config.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.LOCAL_FS_URI.toString());
+
+ exception.expect(FileSystemException.class);
+ exception.
+ expectMessage("Only HDFS is supported as underlyingFileSystem, "
+ + "fs scheme:file");
+ // Start nfs
+ final Nfs3 nfsServer = new Nfs3(config);
+ nfsServer.startServiceInternal(false);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-7454.
RMAppAttemptMetrics#getAggregateResourceUsage can NPE due to double lookup.
Contributed by Jason Lowe.
Posted by su...@apache.org.
YARN-7454. RMAppAttemptMetrics#getAggregateResourceUsage can NPE due to double lookup. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a72c2f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a72c2f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a72c2f5
Branch: refs/heads/YARN-5881
Commit: 0a72c2f56c37063609de72eef1f74632890c048a
Parents: 3a3566e
Author: bibinchundatt <bi...@apache.org>
Authored: Thu Nov 9 21:01:19 2017 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Thu Nov 9 21:01:19 2017 +0530
----------------------------------------------------------------------
.../rmapp/attempt/RMAppAttemptMetrics.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a72c2f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 0982ef9..015cff7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -135,20 +135,21 @@ public class RMAppAttemptMetrics {
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
if (rmApp != null) {
- RMAppAttempt currentAttempt = rmContext.getRMApps().get(attemptId.getApplicationId()).getCurrentAppAttempt();
- if (currentAttempt.getAppAttemptId().equals(attemptId)) {
+ RMAppAttempt currentAttempt = rmApp.getCurrentAppAttempt();
+ if (currentAttempt != null
+ && currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
rmContext.getScheduler().getAppResourceUsageReport(attemptId);
if (appResUsageReport != null) {
Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- if (resourcesUsed.containsKey(entry.getKey())) {
- Long value = resourcesUsed.get(entry.getKey());
+ Long value = resourcesUsed.get(entry.getKey());
+ if (value != null) {
value += entry.getValue();
- resourcesUsed.put(entry.getKey(), value);
- } else{
- resourcesUsed.put(entry.getKey(), entry.getValue());
+ } else {
+ value = entry.getValue();
}
+ resourcesUsed.put(entry.getKey(), value);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-7406. Moving logging APIs
over to slf4j in hadoop-yarn-api. Contributed by Yeliang Cang.
Posted by su...@apache.org.
YARN-7406. Moving logging APIs over to slf4j in hadoop-yarn-api. Contributed by Yeliang Cang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c2b7a36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c2b7a36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c2b7a36
Branch: refs/heads/YARN-5881
Commit: 2c2b7a3672e0744ce6a77a117cedefba04fed603
Parents: 796a0d3
Author: bibinchundatt <bi...@apache.org>
Authored: Sat Nov 11 10:36:27 2017 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Sat Nov 11 10:36:27 2017 +0530
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/conf/HAUtil.java | 12 ++++++------
.../apache/hadoop/yarn/util/resource/ResourceUtils.java | 8 ++++----
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2b7a36/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index 528b642..60c370b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.conf;
import java.net.InetSocketAddress;
import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -30,10 +28,12 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class HAUtil {
- private static Log LOG = LogFactory.getLog(HAUtil.class);
+ private static Logger LOG = LoggerFactory.getLogger(HAUtil.class);
@VisibleForTesting
public static final String BAD_CONFIG_MESSAGE_PREFIX =
@@ -302,9 +302,9 @@ public class HAUtil {
String confKey = getConfKeyForRMInstance(prefix, conf);
String retVal = conf.getTrimmed(confKey);
if (LOG.isTraceEnabled()) {
- LOG.trace("getConfValueForRMInstance: prefix = " + prefix +
- "; confKey being looked up = " + confKey +
- "; value being set to = " + retVal);
+ LOG.trace("getConfValueForRMInstance: prefix = {};" +
+ " confKey being looked up = {};" +
+ " value being set to = {}", prefix, confKey, retVal);
}
return retVal;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c2b7a36/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 9c9c0ef8..1170c72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.util.resource;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -32,6 +30,8 @@ import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -71,7 +71,7 @@ public class ResourceUtils {
private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
private static volatile int numKnownResourceTypes = -1;
- static final Log LOG = LogFactory.getLog(ResourceUtils.class);
+ static final Logger LOG = LoggerFactory.getLogger(ResourceUtils.class);
private ResourceUtils() {
}
@@ -396,7 +396,7 @@ public class ResourceUtils {
} catch (FileNotFoundException fe) {
LOG.info("Unable to find '" + resourceFile + "'.");
} catch (IOException | YarnException ex) {
- LOG.fatal("Exception trying to read resource types configuration '"
+ LOG.error("Exception trying to read resource types configuration '"
+ resourceFile + "'.", ex);
throw new YarnRuntimeException(ex);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-7254. UI and metrics changes
related to absolute resource configuration. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60bef628
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60bef628
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60bef628
Branch: refs/heads/YARN-5881
Commit: 60bef62841c711c0ab87482bd606258441d2adae
Parents: 2a28afb
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 16 11:04:05 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Sat Nov 11 16:24:45 2017 +0530
----------------------------------------------------------------------
.../yarn/api/records/QueueConfigurations.java | 76 +++++++++++
.../yarn/api/records/ResourceInformation.java | 6 +
.../src/main/proto/yarn_protos.proto | 4 +
.../impl/pb/QueueConfigurationsPBImpl.java | 136 ++++++++++++++++++-
.../resource/DominantResourceCalculator.java | 24 +++-
.../scheduler/QueueResourceQuotas.java | 38 ------
.../scheduler/capacity/AbstractCSQueue.java | 18 ++-
.../scheduler/capacity/CSQueue.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/capacity/ParentQueue.java | 56 ++++++--
.../scheduler/capacity/UsersManager.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 23 +++-
.../webapp/CapacitySchedulerPage.java | 49 +++++--
.../webapp/dao/CapacitySchedulerInfo.java | 3 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 6 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 10 +-
.../dao/PartitionQueueCapacitiesInfo.java | 34 ++++-
.../webapp/dao/QueueCapacitiesInfo.java | 13 +-
...CapacitySchedulerWithMultiResourceTypes.java | 25 ++--
.../scheduler/capacity/TestLeafQueue.java | 24 +++-
20 files changed, 443 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
index e25c8aa..30096bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
@@ -147,4 +147,80 @@ public abstract class QueueConfigurations {
@Private
@Unstable
public abstract void setMaxAMPercentage(float maxAMPercentage);
+
+ /**
+ * Get the effective minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMinCapacity();
+
+ /**
+ * Set the effective minimum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * minimum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMinCapacity(Resource capacity);
+
+ /**
+ * Get the effective maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMaxCapacity();
+
+ /**
+ * Set the effective maximum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * maximum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMaxCapacity(Resource capacity);
+
+ /**
+ * Get the configured minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMinCapacity();
+
+ /**
+ * Set the configured minimum capacity of queue (from absolute resource).
+ *
+ * @param configuredMinResource
+ * minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMinCapacity(Resource configuredMinResource);
+
+ /**
+ * Get the configured maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMaxCapacity();
+
+ /**
+ * Set the configured maximum capacity of queue (from absolute resource).
+ *
+ * @param configuredMaxResource
+ * maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 59908ef..05cd05c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -220,6 +220,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
Long.MAX_VALUE);
}
+ public static ResourceInformation newInstance(String name, String units,
+ long minRes, long maxRes) {
+ return ResourceInformation.newInstance(name, units, 0L,
+ ResourceTypes.COUNTABLE, minRes, maxRes);
+ }
+
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index e69c07b..732c01e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -543,6 +543,10 @@ message QueueConfigurationsProto {
optional float maxCapacity = 3;
optional float absoluteMaxCapacity = 4;
optional float maxAMPercentage = 5;
+ optional ResourceProto effectiveMinCapacity = 6;
+ optional ResourceProto effectiveMaxCapacity = 7;
+ optional ResourceProto configuredMinCapacity = 8;
+ optional ResourceProto configuredMaxCapacity = 9;
}
message QueueConfigurationsMapProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index f308bce..80ef4b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -19,16 +19,22 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import com.google.protobuf.TextFormat;
public class QueueConfigurationsPBImpl extends QueueConfigurations {
- QueueConfigurationsProto proto =
- QueueConfigurationsProto.getDefaultInstance();
+ QueueConfigurationsProto proto = QueueConfigurationsProto
+ .getDefaultInstance();
QueueConfigurationsProto.Builder builder = null;
+ Resource configuredMinResource = null;
+ Resource configuredMaxResource = null;
+ Resource effMinResource = null;
+ Resource effMaxResource = null;
boolean viaProto = false;
public QueueConfigurationsPBImpl() {
@@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
}
public QueueConfigurationsProto getProto() {
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.effMinResource != null) {
+ builder
+ .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource));
+ }
+ if (this.effMaxResource != null) {
+ builder
+ .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
+ }
+ if (this.configuredMinResource != null) {
+ builder.setEffectiveMinCapacity(
+ convertToProtoFormat(this.configuredMinResource));
+ }
+ if (this.configuredMaxResource != null) {
+ builder.setEffectiveMaxCapacity(
+ convertToProtoFormat(this.configuredMaxResource));
+ }
+ }
+
@Override
public float getCapacity() {
QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
@@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
builder.setMaxAMPercentage(maxAMPercentage);
}
+ @Override
+ public Resource getEffectiveMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMinResource != null) {
+ return this.effMinResource;
+ }
+ if (!p.hasEffectiveMinCapacity()) {
+ return null;
+ }
+ this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity());
+ return this.effMinResource;
+ }
+
+ @Override
+ public void setEffectiveMinCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMinCapacity();
+ }
+ this.effMinResource = capacity;
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMaxResource != null) {
+ return this.effMaxResource;
+ }
+ if (!p.hasEffectiveMaxCapacity()) {
+ return null;
+ }
+ this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity());
+ return this.effMaxResource;
+ }
+
+ @Override
+ public void setEffectiveMaxCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMaxCapacity();
+ }
+ this.effMaxResource = capacity;
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ProtoUtils.convertToProtoFormat(t);
+ }
+
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = QueueConfigurationsProto.newBuilder(proto);
@@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
return false;
}
+ @Override
+ public Resource getConfiguredMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMinResource != null) {
+ return this.configuredMinResource;
+ }
+ if (!p.hasConfiguredMinCapacity()) {
+ return null;
+ }
+ this.configuredMinResource = convertFromProtoFormat(
+ p.getConfiguredMinCapacity());
+ return this.configuredMinResource;
+ }
+
+ @Override
+ public void setConfiguredMinCapacity(Resource configuredMinResource) {
+ maybeInitBuilder();
+ if (configuredMinResource == null) {
+ builder.clearConfiguredMinCapacity();
+ }
+ this.configuredMinResource = configuredMinResource;
+ }
+
+ @Override
+ public Resource getConfiguredMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMaxResource != null) {
+ return this.configuredMaxResource;
+ }
+ if (!p.hasConfiguredMaxCapacity()) {
+ return null;
+ }
+ this.configuredMaxResource = convertFromProtoFormat(
+ p.getConfiguredMaxCapacity());
+ return this.configuredMaxResource;
+ }
+
+ @Override
+ public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+ maybeInitBuilder();
+ if (configuredMaxResource == null) {
+ builder.clearConfiguredMaxCapacity();
+ }
+ this.configuredMaxResource = configuredMaxResource;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 16e4527..cc4acf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
- return Resources.createResource(
- roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
- roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ Resource ret = Resource.newInstance(r);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation rResourceInformation = r.getResourceInformation(i);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(i);
+ ResourceInformation tmp = ret.getResourceInformation(i);
+
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+
+ long value = rValue;
+ if (stepFactorValue != 0) {
+ value = roundDown(rValue, stepFactorValue);
+ }
+ tmp.setValue(value);
+ }
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
index 2e653fc..08b4d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage {
public void setEffectiveMaxResource(String label, Resource res) {
_set(label, ResourceType.EFF_MAX_RESOURCE, res);
}
-
- /*
- * Effective Minimum Resource
- */
- public Resource getEffectiveMinResourceUp() {
- return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public Resource getEffectiveMinResourceUp(String label) {
- return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public void setEffectiveMinResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- public void setEffectiveMinResourceUp(Resource res) {
- _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- /*
- * Effective Maximum Resource
- */
- public Resource getEffectiveMaxResourceUp() {
- return getEffectiveMaxResourceUp(NL);
- }
-
- public Resource getEffectiveMaxResourceUp(String label) {
- return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
- }
-
- public void setEffectiveMaxResourceUp(Resource res) {
- setEffectiveMaxResourceUp(NL, res);
- }
-
- public void setEffectiveMaxResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index b2539c5..c2aca0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResource = conf.getMaximumResourceRequirement(label,
queuePath, resourceTypes);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
this.capacityConfigType = (!minResource.equals(Resources.none())
&& queueCapacities.getAbsoluteCapacity(label) == 0f)
@@ -481,12 +485,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public Resource getEffectiveCapacityUp(String label) {
- return Resources
- .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
- }
-
- @Override
public Resource getEffectiveCapacityDown(String label, Resource factor) {
return Resources.normalizeDown(resourceCalculator,
getQueueResourceQuotas().getEffectiveMinResource(label),
@@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue {
queueConfiguration.setMaxCapacity(maxCapacity);
queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+ queueConfiguration.setConfiguredMinCapacity(
+ queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+ queueConfiguration.setConfiguredMaxCapacity(
+ queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+ queueConfiguration.setEffectiveMinCapacity(
+ queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+ queueConfiguration.setEffectiveMaxCapacity(
+ queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
queueConfigurations.put(nodeLabel, queueConfiguration);
}
return queueConfigurations;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 2e29a71..624fcc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.security.AccessControlException;
@@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* <code>CSQueue</code> represents a node in the tree of
* hierarchical queues in the {@link CapacityScheduler}.
@@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return effective queue capacity
*/
Resource getEffectiveCapacity(String label);
- Resource getEffectiveCapacityUp(String label);
Resource getEffectiveCapacityDown(String label, Resource factor);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1460121..41ec4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index d5a1354..75d7641 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue {
queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ queueCapacities.setAbsoluteCapacity(label, 1.0f);
}
- // Total configured min resources of direct children of queue
+ // Total configured min resources of direct children of this given parent
+ // queue.
Resource configuredMinResources = Resource.newInstance(0L, 0);
for (CSQueue childQueue : getChildQueues()) {
Resources.addTo(configuredMinResources,
@@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue {
// present could also be taken from effective max resource of parent.
Resource parentMaxRes = queueResourceQuotas
.getConfiguredMaxResource(label);
- if (parentMaxRes.equals(Resources.none())) {
+ if (parent != null && parentMaxRes.equals(Resources.none())) {
parentMaxRes = parent.getQueueResourceQuotas()
.getEffectiveMaxResource(label);
}
@@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue {
parentMaxRes);
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.clone(effMaxResource));
+
+ // In cases where we still need to update some units based on
+ // percentage, we have to calculate percentage and update.
+ deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc,
+ childQueue);
} else {
childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
Resources.multiply(resourceByLabel,
@@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue {
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
.getAbsoluteMaximumCapacity(label)));
-
- childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
- childQueue.getQueueCapacities().getAbsoluteCapacity(label),
- minimumAllocation));
- childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc,
- resourceByLabel, childQueue.getQueueCapacities()
- .getAbsoluteMaximumCapacity(label),
- minimumAllocation));
}
if (LOG.isDebugEnabled()) {
@@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private void deriveCapacityFromAbsoluteConfigurations(String label,
+ Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
+
+ /*
+ * In case when queues are configured with absolute resources, it is better
+ * to update capacity/max-capacity etc w.r.t absolute resource as well. In
+ * case of computation, these values wont be used any more. However for
+ * metrics and UI, its better these values are pre-computed here itself.
+ */
+
+ // 1. Update capacity as a float based on parent's minResource
+ childQueue.getQueueCapacities().setCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMinResource(label),
+ getQueueResourceQuotas().getEffectiveMinResource(label)));
+
+ // 2. Update max-capacity as a float based on parent's maxResource
+ childQueue.getQueueCapacities().setMaximumCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label),
+ getQueueResourceQuotas().getEffectiveMaxResource(label)));
+
+ // 3. Update absolute capacity as a float based on parent's minResource and
+ // cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteCapacity(label,
+ (float) childQueue.getQueueCapacities().getCapacity()
+ / getQueueCapacities().getAbsoluteCapacity(label));
+
+ // 4. Update absolute max-capacity as a float based on parent's maxResource
+ // and cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
+ (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
+ / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index efc20e9..7287c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
+ Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 40405fc..502157f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -903,6 +904,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
StringBuilder diagnosticMessage) {
LeafQueue queue = getCSLeafQueue();
QueueCapacities queueCapacities = queue.getQueueCapacities();
+ QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
diagnosticMessage.append(" Details : AM Partition = ");
diagnosticMessage.append(appAMNodePartitionName.isEmpty()
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
@@ -924,6 +926,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
* 100);
diagnosticMessage.append(" % ; ");
+ diagnosticMessage.append("Queue's capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's used capacity (absolute resource) = ");
+ diagnosticMessage
+ .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's max capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
}
/**
@@ -987,13 +1001,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
if (!calc.isInvalidDivisor(totalPartitionRes)) {
- float queueAbsMaxCapPerPartition =
- ((AbstractCSQueue) getQueue()).getQueueCapacities()
- .getAbsoluteCapacity(getAppAMNodePartitionName());
+ Resource effCap = ((AbstractCSQueue) getQueue())
+ .getEffectiveCapacity(getAppAMNodePartitionName());
float queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(),
- Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
- * 100;
+ report.getUsedResources(), effCap) * 100;
report.setQueueUsagePercentage(queueUsagePerc);
}
return report;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 74b4e79..3f68e07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView {
? new ResourceInfo(Resources.none())
: resourceUsages.getAmUsed();
ri.
- __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
- __("Configured Capacity:", percent(capacities.getCapacity() / 100)).
- __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)).
- __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
- __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
- __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
- __("Used Resources:", resourceUsages.getUsed().toString()).
- __("Configured Max Application Master Limit:", StringUtils.format("%.1f",
- capacities.getMaxAMLimitPercentage())).
- __("Max Application Master Resources:",
- resourceUsages.getAMLimit().toString()).
- __("Used Application Master Resources:",
- amUsed.toString()).
- __("Max Application Master Resources Per User:",
- userAMResourceLimit.toString());
+ __("Used Capacity:",
+ appendPercent(resourceUsages.getUsed().toString(),
+ capacities.getUsedCapacity() / 100))
+ .__("Configured Capacity:",
+ capacities.getConfiguredMinResource().toString())
+ .__("Configured Max Capacity:",
+ capacities.getConfiguredMaxResource().getResource()
+ .equals(Resources.none())
+ ? "unlimited"
+ : capacities.getConfiguredMaxResource().toString())
+ .__("Effective Capacity:",
+ appendPercent(capacities.getEffectiveMinResource().toString(),
+ capacities.getCapacity() / 100))
+ .__("Effective Max Capacity:",
+ appendPercent(capacities.getEffectiveMaxResource().toString(),
+ capacities.getMaxCapacity() / 100))
+ .__("Absolute Used Capacity:",
+ percent(capacities.getAbsoluteUsedCapacity() / 100))
+ .__("Absolute Configured Capacity:",
+ percent(capacities.getAbsoluteCapacity() / 100))
+ .__("Absolute Configured Max Capacity:",
+ percent(capacities.getAbsoluteMaxCapacity() / 100))
+ .__("Used Resources:", resourceUsages.getUsed().toString())
+ .__("Configured Max Application Master Limit:",
+ StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage()))
+ .__("Max Application Master Resources:",
+ resourceUsages.getAMLimit().toString())
+ .__("Used Application Master Resources:", amUsed.toString())
+ .__("Max Application Master Resources Per User:",
+ userAMResourceLimit.toString());
}
private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView {
return QueuesBlock.class;
}
+ static String appendPercent(String message, float f) {
+ return message + " (" + StringUtils.formatPercent(f, 1) + ")";
+ }
+
static String percent(float f) {
return StringUtils.formatPercent(f, 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 32e4ac5..ca42041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
max = 1f;
this.maxCapacity = max * 100;
- capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
+ capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
+ parent.getQueueResourceQuotas(), false);
queues = getQueues(parent);
health = new CapacitySchedulerHealthInfo(cs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 7dcdf58..343c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
}
@Override
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas);
}
public int getNumActiveApplications() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 86b2fea..d4de9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo {
Collections.sort(nodeLabels);
}
QueueCapacities qCapacities = q.getQueueCapacities();
- populateQueueCapacities(qCapacities);
+ QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas();
+ populateQueueCapacities(qCapacities, qResQuotas);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
@@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo {
resources = new ResourcesInfo(queueResourceUsage, false);
}
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities, false);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas,
+ false);
}
public float getCapacity() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
index 5e298f9..2a15502 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
@@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
/**
* This class represents queue capacities for a given partition
*/
@@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo {
private float absoluteUsedCapacity;
private float absoluteMaxCapacity = 100;
private float maxAMLimitPercentage;
+ private ResourceInfo configuredMinResource;
+ private ResourceInfo configuredMaxResource;
+ private ResourceInfo effectiveMinResource;
+ private ResourceInfo effectiveMaxResource;
public PartitionQueueCapacitiesInfo() {
}
public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
float usedCapacity, float maxCapacity, float absCapacity,
- float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) {
+ float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
+ Resource confMinRes, Resource confMaxRes, Resource effMinRes,
+ Resource effMaxRes) {
super();
this.partitionName = partitionName;
this.capacity = capacity;
@@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo {
this.absoluteUsedCapacity = absUsedCapacity;
this.absoluteMaxCapacity = absMaxCapacity;
this.maxAMLimitPercentage = maxAMLimitPercentage;
+ this.configuredMinResource = new ResourceInfo(confMinRes);
+ this.configuredMaxResource = new ResourceInfo(confMaxRes);
+ this.effectiveMinResource = new ResourceInfo(effMinRes);
+ this.effectiveMaxResource = new ResourceInfo(effMaxRes);
}
public float getCapacity() {
@@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo {
public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
this.maxAMLimitPercentage = maxAMLimitPercentage;
}
+
+ public ResourceInfo getConfiguredMinResource() {
+ return configuredMinResource;
+ }
+
+ public ResourceInfo getConfiguredMaxResource() {
+ if (configuredMaxResource.getResource().equals(Resources.none())) {
+ return null;
+ }
+ return configuredMaxResource;
+ }
+
+ public ResourceInfo getEffectiveMinResource() {
+ return effectiveMinResource;
+ }
+
+ public ResourceInfo getEffectiveMaxResource() {
+ return effectiveMaxResource;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
index 9a3e439..35c80d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
/**
@@ -39,6 +40,7 @@ public class QueueCapacitiesInfo {
}
public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas,
boolean considerAMUsage) {
if (capacities == null) {
return;
@@ -68,12 +70,17 @@ public class QueueCapacitiesInfo {
queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
absUsedCapacity, absMaxCapacity,
- considerAMUsage ? maxAMLimitPercentage : 0f));
+ considerAMUsage ? maxAMLimitPercentage : 0f,
+ resourceQuotas.getConfiguredMinResource(partitionName),
+ resourceQuotas.getConfiguredMaxResource(partitionName),
+ resourceQuotas.getEffectiveMinResource(partitionName),
+ resourceQuotas.getEffectiveMaxResource(partitionName)));
}
}
- public QueueCapacitiesInfo(QueueCapacities capacities) {
- this(capacities, true);
+ public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas) {
+ this(capacities, resourceQuotas, true);
}
public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index b386c18..38768e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
Map<String, ResourceInformation> riMap = new HashMap<>();
// Initialize mandatory resources
- riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
- riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
- riMap.put(RESOURCE_1, ResourceInformation
- .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
- Integer.MAX_VALUE));
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+ ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
@@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
- Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+ Assert.assertEquals(Resource.newInstance(1 * GB, 1),
leafQueue.getUsedResources());
RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
// Do node heartbeats 1 time and check container allocated.
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+ // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
Assert.assertEquals(
- TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+ TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
leafQueue.getUsedResources());
// Acquire container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60bef628/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 96f8970..30bff78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -1355,7 +1356,7 @@ public class TestLeafQueue {
// TODO, fix headroom in the future patch
assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
// User limit = 2G, 2 in use
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+ assertEquals(1*GB, app_1.getHeadroom().getMemorySize());
// the application is not yet active
// Again one to user_0 since he hasn't exceeded user limit yet
@@ -1366,8 +1367,8 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemorySize());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+ assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
@@ -3874,7 +3875,7 @@ public class TestLeafQueue {
// Queue "test" consumes 100% of the cluster, so its capacity and absolute
// capacity are both 1.0f.
- Queue queue = createQueue("test", null, 1.0f, 1.0f);
+ Queue queue = createQueue("test", null, 1.0f, 1.0f, res);
final String user = "user1";
FiCaSchedulerApp app =
new FiCaSchedulerApp(appAttId, user, queue,
@@ -3891,7 +3892,8 @@ public class TestLeafQueue {
// Queue "test2" is a child of root and its capacity is 50% of root. As a
// child of root, its absolute capaicty is also 50%.
- queue = createQueue("test2", null, 0.5f, 0.5f);
+ queue = createQueue("test2", null, 0.5f, 0.5f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 2));
app = new FiCaSchedulerApp(appAttId, user, queue,
queue.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3903,7 +3905,8 @@ public class TestLeafQueue {
// Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
// Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
- AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+ AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 4));
app = new FiCaSchedulerApp(appAttId, user, qChild,
qChild.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3922,7 +3925,7 @@ public class TestLeafQueue {
}
private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
- float absCap) {
+ float absCap, Resource res) {
CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
null, QueueState.RUNNING, null, "", null, false);
@@ -3934,6 +3937,13 @@ public class TestLeafQueue {
QueueCapacities qCaps = mock(QueueCapacities.class);
when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
when(queue.getQueueCapacities()).thenReturn(qCaps);
+ QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class);
+ when(qQuotas.getConfiguredMinResource(any())).thenReturn(res);
+ when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMinResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res);
+ when(queue.getQueueResourceQuotas()).thenReturn(qQuotas);
+ when(queue.getEffectiveCapacity(any())).thenReturn(res);
return queue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-7166. Container REST
endpoints should report resource types
Posted by su...@apache.org.
YARN-7166. Container REST endpoints should report resource types
Change-Id: If9c2fe58d4cf758bb6b6cf363dc01f35f8720987
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0de10680
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0de10680
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0de10680
Branch: refs/heads/YARN-5881
Commit: 0de10680b7e5a9dfc85173bcfd338fd3656aa57f
Parents: cb35a59
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Nov 8 16:43:49 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Nov 8 16:43:49 2017 -0800
----------------------------------------------------------------------
.../yarn/server/webapp/dao/ContainerInfo.java | 39 +++++++++++++++++---
1 file changed, 34 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0de10680/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
index 1a5ee85..26a822c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.webapp.dao;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@@ -27,6 +30,8 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.util.Times;
@Public
@@ -49,20 +54,18 @@ public class ContainerInfo {
protected ContainerState containerState;
protected String nodeHttpAddress;
protected String nodeId;
+ protected Map<String, Long> allocatedResources;
public ContainerInfo() {
// JAXB needs this
}
public ContainerInfo(ContainerReport container) {
- containerId = container.getContainerId().toString();
- if (container.getAllocatedResource() != null) {
- allocatedMB = container.getAllocatedResource().getMemorySize();
- allocatedVCores = container.getAllocatedResource().getVirtualCores();
- }
if (container.getAssignedNode() != null) {
assignedNodeId = container.getAssignedNode().toString();
}
+
+ containerId = container.getContainerId().toString();
priority = container.getPriority().getPriority();
startedTime = container.getCreationTime();
finishedTime = container.getFinishTime();
@@ -73,6 +76,22 @@ public class ContainerInfo {
containerState = container.getContainerState();
nodeHttpAddress = container.getNodeHttpAddress();
nodeId = container.getAssignedNode().toString();
+
+ Resource allocated = container.getAllocatedResource();
+
+ if (allocated != null) {
+ allocatedMB = allocated.getMemorySize();
+ allocatedVCores = allocated.getVirtualCores();
+
+ // Now populate the allocated resources. This map will include memory
+ // and CPU, because it's where they belong. We still keep allocatedMB
+ // and allocatedVCores so that we don't break the API.
+ allocatedResources = new HashMap<>();
+
+ for (ResourceInformation info : allocated.getResources()) {
+ allocatedResources.put(info.getName(), info.getValue());
+ }
+ }
}
public String getContainerId() {
@@ -130,4 +149,14 @@ public class ContainerInfo {
public String getNodeId() {
return nodeId;
}
+
+ /**
+ * Return a map of the allocated resources. The map key is the resource name,
+ * and the value is the resource value.
+ *
+ * @return the allocated resources map
+ */
+ public Map<String, Long> getAllocatedResources() {
+ return Collections.unmodifiableMap(allocatedResources);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-12498. Journal Syncer is not
started in Federated + HA cluster. Contributed by Bharat Viswanadham.
Posted by su...@apache.org.
HDFS-12498. Journal Syncer is not started in Federated + HA cluster. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d201f77
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d201f77
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d201f77
Branch: refs/heads/YARN-5881
Commit: 6d201f77c734d6c6a9e3e297fe3dbff251cbb8b3
Parents: 1d6f8be
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Nov 10 16:30:38 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Nov 10 16:30:38 2017 -0800
----------------------------------------------------------------------
.../hdfs/qjournal/server/JournalNodeSyncer.java | 55 ++++++++--
.../hdfs/qjournal/server/TestJournalNode.java | 103 ++++++++++++++++++-
2 files changed, 146 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d201f77/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
index cf5a9ec..490b3ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos
@@ -51,6 +52,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
/**
@@ -263,25 +266,63 @@ public class JournalNodeSyncer {
}
private List<InetSocketAddress> getOtherJournalNodeAddrs() {
- URI uri = null;
+ String uriStr = "";
try {
- String uriStr = conf.get(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+ uriStr = conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
+
+ if (uriStr == null || uriStr.isEmpty()) {
+ if (nameServiceId != null) {
+ uriStr = conf.getTrimmed(DFSConfigKeys
+ .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "." + nameServiceId);
+ }
+ }
+
if (uriStr == null || uriStr.isEmpty()) {
- LOG.warn("Could not construct Shared Edits Uri");
+ HashSet<String> sharedEditsUri = Sets.newHashSet();
+ if (nameServiceId != null) {
+ Collection<String> nnIds = DFSUtilClient.getNameNodeIds(
+ conf, nameServiceId);
+ for (String nnId : nnIds) {
+ String suffix = nameServiceId + "." + nnId;
+ uriStr = conf.getTrimmed(DFSConfigKeys
+ .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + "." + suffix);
+ sharedEditsUri.add(uriStr);
+ }
+ if (sharedEditsUri.size() > 1) {
+ uriStr = null;
+ LOG.error("The conf property " + DFSConfigKeys
+ .DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " not set properly, " +
+ "it has been configured with different journalnode values " +
+ sharedEditsUri.toString() + " for a" +
+ " single nameserviceId" + nameServiceId);
+ }
+ }
+ }
+
+ if (uriStr == null || uriStr.isEmpty()) {
+ LOG.error("Could not construct Shared Edits Uri");
return null;
+ } else {
+ return getJournalAddrList(uriStr);
}
- uri = new URI(uriStr);
- return Util.getLoggerAddresses(uri,
- Sets.newHashSet(jn.getBoundIpcAddress()));
+
} catch (URISyntaxException e) {
LOG.error("The conf property " + DFSConfigKeys
.DFS_NAMENODE_SHARED_EDITS_DIR_KEY + " not set properly.");
} catch (IOException e) {
- LOG.error("Could not parse JournalNode addresses: " + uri);
+ LOG.error("Could not parse JournalNode addresses: " + uriStr);
}
return null;
}
+ private List<InetSocketAddress> getJournalAddrList(String uriStr) throws
+ URISyntaxException,
+ IOException {
+ URI uri = new URI(uriStr);
+ return Util.getLoggerAddresses(uri,
+ Sets.newHashSet(jn.getBoundIpcAddress()));
+ }
+
private JournalIdProto convertJournalId(String journalId) {
return QJournalProtocolProtos.JournalIdProto.newBuilder()
.setIdentifier(journalId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d201f77/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
index 5bf69f0..9bd686f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java
@@ -106,6 +106,24 @@ public class TestJournalNode {
"testJournalNodeSyncerNotStartWhenSyncEnabled")) {
conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
"qjournal://jn0:9900;jn1:9901");
+ } else if (testName.getMethodName().equals(
+ "testJournalNodeSyncwithFederationTypeConfigWithNameServiceId")) {
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1",
+ "qjournal://journalnode0:9900;journalnode0:9901");
+ } else if (testName.getMethodName().equals(
+ "testJournalNodeSyncwithFederationTypeConfigWithNamenodeId")) {
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".ns1", "nn1,nn2");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn1",
+ "qjournal://journalnode0:9900;journalnode1:9901");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn2",
+ "qjournal://journalnode0:9900;journalnode1:9901");
+ } else if (testName.getMethodName().equals(
+ "testJournalNodeSyncwithFederationTypeIncorrectConfigWithNamenodeId")) {
+ conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".ns1", "nn1,nn2");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn1",
+ "qjournal://journalnode0:9900;journalnode1:9901");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY +".ns1" +".nn2",
+ "qjournal://journalnode0:9902;journalnode1:9903");
}
jn = new JournalNode();
jn.setConf(conf);
@@ -387,7 +405,7 @@ public class TestJournalNode {
@Test
public void testJournalNodeSyncerNotStartWhenSyncDisabled()
- throws IOException{
+ throws IOException {
//JournalSyncer will not be started, as journalsync is not enabled
conf.setBoolean(DFSConfigKeys.DFS_JOURNALNODE_ENABLE_SYNC_KEY, false);
jn.getOrCreateJournal(journalId);
@@ -408,7 +426,7 @@ public class TestJournalNode {
@Test
public void testJournalNodeSyncerNotStartWhenSyncEnabledIncorrectURI()
- throws IOException{
+ throws IOException {
//JournalSyncer will not be started,
// as shared edits hostnames are not resolved
jn.getOrCreateJournal(journalId);
@@ -431,7 +449,7 @@ public class TestJournalNode {
@Test
public void testJournalNodeSyncerNotStartWhenSyncEnabled()
- throws IOException{
+ throws IOException {
//JournalSyncer will not be started,
// as shared edits hostnames are not resolved
jn.getOrCreateJournal(journalId);
@@ -452,9 +470,84 @@ public class TestJournalNode {
}
- private void setupStaticHostResolution(int nameServiceIdCount,
+
+ @Test
+ public void testJournalNodeSyncwithFederationTypeConfigWithNameServiceId()
+ throws IOException {
+ //JournalSyncer will not be started, as nameserviceId passed is null,
+ // but configured shared edits dir is appended with nameserviceId
+ setupStaticHostResolution(2, "journalnode");
+ jn.getOrCreateJournal(journalId);
+ Assert.assertEquals(false,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(false,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+ //Trying by passing nameserviceId and resolve hostnames
+ // now IstriedJournalSyncerStartWithnsId should be set
+ // and also journalnode syncer will also be started
+
+ jn.getOrCreateJournal(journalId, "ns1");
+ Assert.assertEquals(true,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(true,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+ }
+
+ @Test
+ public void testJournalNodeSyncwithFederationTypeConfigWithNamenodeId()
+ throws IOException {
+ //JournalSyncer will not be started, as nameserviceId passed is null,
+ // but configured shared edits dir is appended with nameserviceId +
+ // namenodeId
+ setupStaticHostResolution(2, "journalnode");
+ jn.getOrCreateJournal(journalId);
+ Assert.assertEquals(false,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(false,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+ //Trying by passing nameserviceId and resolve hostnames
+ // now IstriedJournalSyncerStartWithnsId should be set
+ // and also journalnode syncer will also be started
+
+ jn.getOrCreateJournal(journalId, "ns1");
+ Assert.assertEquals(true,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(true,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+ }
+
+ @Test
+ public void
+ testJournalNodeSyncwithFederationTypeIncorrectConfigWithNamenodeId()
+ throws IOException {
+ //JournalSyncer will not be started, as nameserviceId passed is null,
+ // but configured shared edits dir is appended with nameserviceId +
+ // namenodeId
+ setupStaticHostResolution(2, "journalnode");
+ jn.getOrCreateJournal(journalId);
+ Assert.assertEquals(false,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(false,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+
+ //Trying by passing nameserviceId and resolve hostnames
+ // now IstriedJournalSyncerStartWithnsId should be set
+ // and journalnode syncer will not be started
+ // as for each nnId, different shared Edits dir value is configured
+
+ jn.getOrCreateJournal(journalId, "ns1");
+ Assert.assertEquals(false,
+ jn.getJournalSyncerStatus(journalId));
+ Assert.assertEquals(true,
+ jn.getJournal(journalId).getTriedJournalSyncerStartedwithnsId());
+ }
+
+
+ private void setupStaticHostResolution(int journalNodeCount,
String hostname) {
- for (int i = 0; i < nameServiceIdCount; i++) {
+ for (int i = 0; i < journalNodeCount; i++) {
NetUtils.addStaticResolution(hostname + i,
"localhost");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-14929. Cleanup usage of
decodecomponent and use QueryStringDecoder from netty. Contributed by Bharat
Viswanadham.
Posted by su...@apache.org.
HADOOP-14929. Cleanup usage of decodecomponent and use QueryStringDecoder from netty. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d6f8beb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d6f8beb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d6f8beb
Branch: refs/heads/YARN-5881
Commit: 1d6f8bebe9d20c958e419c140109e3d9fec8cb46
Parents: 8a1bd9a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Nov 10 16:28:12 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Nov 10 16:28:12 2017 -0800
----------------------------------------------------------------------
.../datanode/web/webhdfs/ParameterParser.java | 66 ++--------------
.../web/webhdfs/TestParameterParser.java | 81 +++++++++++++++++++-
2 files changed, 87 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d6f8beb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index 16380e5..2b3a393 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.security.token.Token;
import java.io.IOException;
import java.net.URI;
-import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.EnumSet;
import java.util.List;
@@ -143,9 +142,13 @@ class ParameterParser {
}
public EnumSet<CreateFlag> createFlag() {
- String cf =
- decodeComponent(param(CreateFlagParam.NAME), StandardCharsets.UTF_8);
-
+ String cf = "";
+ if (param(CreateFlagParam.NAME) != null) {
+ QueryStringDecoder decoder = new QueryStringDecoder(
+ param(CreateFlagParam.NAME),
+ StandardCharsets.UTF_8);
+ cf = decoder.path();
+ }
return new CreateFlagParam(cf).getValue();
}
@@ -159,61 +162,6 @@ class ParameterParser {
}
/**
- * The following function behaves exactly the same as netty's
- * <code>QueryStringDecoder#decodeComponent</code> except that it
- * does not decode the '+' character as space. WebHDFS takes this scheme
- * to maintain the backward-compatibility for pre-2.7 releases.
- */
- private static String decodeComponent(final String s, final Charset charset) {
- if (s == null) {
- return "";
- }
- final int size = s.length();
- boolean modified = false;
- for (int i = 0; i < size; i++) {
- final char c = s.charAt(i);
- if (c == '%' || c == '+') {
- modified = true;
- break;
- }
- }
- if (!modified) {
- return s;
- }
- final byte[] buf = new byte[size];
- int pos = 0; // position in `buf'.
- for (int i = 0; i < size; i++) {
- char c = s.charAt(i);
- if (c == '%') {
- if (i == size - 1) {
- throw new IllegalArgumentException("unterminated escape sequence at" +
- " end of string: " + s);
- }
- c = s.charAt(++i);
- if (c == '%') {
- buf[pos++] = '%'; // "%%" -> "%"
- break;
- }
- if (i == size - 1) {
- throw new IllegalArgumentException("partial escape sequence at end " +
- "of string: " + s);
- }
- c = decodeHexNibble(c);
- final char c2 = decodeHexNibble(s.charAt(++i));
- if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) {
- throw new IllegalArgumentException(
- "invalid escape sequence `%" + s.charAt(i - 1) + s.charAt(
- i) + "' at index " + (i - 2) + " of: " + s);
- }
- c = (char) (c * 16 + c2);
- // Fall through.
- }
- buf[pos++] = (byte) c;
- }
- return new String(buf, 0, pos, charset);
- }
-
- /**
* Helper to decode half of a hexadecimal number from a string.
* @param c The ASCII character of the hexadecimal number to decode.
* Must be in the range {@code [0-9a-fA-F]}.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d6f8beb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
index 59fd18f..235d051 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HAUtilClient;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -25,14 +26,17 @@ import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
import io.netty.handler.codec.http.QueryStringDecoder;
import java.io.IOException;
+import java.util.EnumSet;
-import static org.mockito.Mockito.mock;
public class TestParameterParser {
private static final String LOGICAL_NAME = "minidfs";
@@ -64,6 +68,81 @@ public class TestParameterParser {
}
@Test
+ public void testCreateFlag() {
+ final String path = "/test1?createflag=append,sync_block";
+ Configuration conf = new Configuration();
+ QueryStringDecoder decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path);
+ ParameterParser testParser = new ParameterParser(decoder, conf);
+ EnumSet<CreateFlag> actual = testParser.createFlag();
+ EnumSet<CreateFlag> expected = EnumSet.of(CreateFlag.APPEND,
+ CreateFlag.SYNC_BLOCK);
+ Assert.assertEquals(expected.toString(), actual.toString());
+
+
+ final String path1 = "/test1?createflag=append";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path1);
+ testParser = new ParameterParser(decoder, conf);
+
+ actual = testParser.createFlag();
+ expected = EnumSet.of(CreateFlag.APPEND);
+ Assert.assertEquals(expected, actual);
+
+ final String path2 = "/test1";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path2);
+ testParser = new ParameterParser(decoder, conf);
+ actual = testParser.createFlag();
+ Assert.assertEquals(0, actual.size());
+
+ final String path3 = "/test1?createflag=create,overwrite";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path3);
+ testParser = new ParameterParser(decoder, conf);
+ actual = testParser.createFlag();
+ expected = EnumSet.of(CreateFlag.CREATE, CreateFlag
+ .OVERWRITE);
+ Assert.assertEquals(expected.toString(), actual.toString());
+
+
+ final String path4 = "/test1?createflag=";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path4);
+ testParser = new ParameterParser(decoder, conf);
+ actual = testParser.createFlag();
+ Assert.assertEquals(0, actual.size());
+
+ //Incorrect value passed to createflag
+ try {
+ final String path5 = "/test1?createflag=overwrite,";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path5);
+ testParser = new ParameterParser(decoder, conf);
+ actual = testParser.createFlag();
+ fail("It should throw Illegal Argument Exception");
+ } catch (Exception e) {
+ GenericTestUtils
+ .assertExceptionContains("No enum constant", e);
+ }
+
+ //Incorrect value passed to createflag
+ try {
+ final String path6 = "/test1?createflag=,";
+ decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + path6);
+ testParser = new ParameterParser(decoder, conf);
+ actual = testParser.createFlag();
+ fail("It should throw Illegal Argument Exception");
+ } catch (Exception e) {
+ GenericTestUtils
+ .assertExceptionContains("No enum constant", e);
+ }
+
+
+ }
+
+ @Test
public void testOffset() throws IOException {
final long X = 42;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-7423. Improve service client
loggings. Contributed by Jian He
Posted by su...@apache.org.
YARN-7423. Improve service client loggings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0
Branch: refs/heads/YARN-5881
Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b
Parents: a6c4bd7
Author: Billie Rinaldi <bi...@apache.org>
Authored: Sat Nov 4 12:27:42 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceMaster.java | 4 +-
.../yarn/service/api/records/Configuration.java | 8 +-
.../yarn/service/client/ServiceClient.java | 50 +-
.../yarn/service/component/Component.java | 10 +-
.../component/instance/ComponentInstance.java | 4 +-
.../containerlaunch/AbstractLauncher.java | 6 +-
.../containerlaunch/ClasspathConstructor.java | 4 +-
.../containerlaunch/CommandLineBuilder.java | 4 +-
.../containerlaunch/JavaCommandLineBuilder.java | 4 +-
.../yarn/service/monitor/probe/HttpProbe.java | 4 +-
.../yarn/service/monitor/probe/PortProbe.java | 4 +-
.../provider/AbstractClientProvider.java | 8 +-
.../provider/AbstractProviderService.java | 4 +-
.../yarn/service/provider/ProviderUtils.java | 10 +-
.../registry/YarnRegistryViewForProviders.java | 6 +-
.../yarn/service/utils/CoreFileSystem.java | 2 +-
.../hadoop/yarn/service/utils/PortScanner.java | 4 +-
.../yarn/service/utils/ServiceApiUtil.java | 2 +-
.../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++++++++++++++
.../hadoop/yarn/service/utils/SliderUtils.java | 564 -------------------
.../yarn/service/utils/ZookeeperUtils.java | 4 +-
.../service/conf/TestValidateServiceNames.java | 1 -
22 files changed, 614 insertions(+), 638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2d86a5a..b0b4f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService {
}
protected ContainerId getAMContainerId() throws BadClusterStateException {
- return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+ return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable(
ApplicationConstants.Environment.CONTAINER_ID.name()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index dc6dff8..859c1ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -24,7 +24,7 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -201,9 +201,9 @@ public class Configuration implements Serializable {
* this ConfigFile.
*/
public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
.getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
Map<String, ConfigFile> thatMap = new HashMap<>();
for (ConfigFile file : that.getFiles()) {
@@ -212,7 +212,7 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
thatFile.getProperties());
thatMap.remove(thisFile.getDestFile());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 775a642..11cd30d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
@@ -80,19 +80,13 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+import static org.apache.hadoop.yarn.service.utils.ServiceUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -169,7 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
fileName);
}
Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading service definition from: " + filePath);
+ LOG.info("Loading service definition from local FS: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (!StringUtils.isEmpty(serviceName)) {
@@ -482,7 +476,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
// though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
+ if (ServiceUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the" + " configuration property "
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -505,7 +499,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
if (serviceName != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+ tags = Collections.singleton(ServiceUtils.createNameTag(serviceName));
}
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
request.setApplicationTypes(types);
@@ -652,7 +646,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
// write out the path output
CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
String cmdStr = CLI.build();
- LOG.info("AM launch command: {}", cmdStr);
+ LOG.debug("AM launch command: {}", cmdStr);
return cmdStr;
}
@@ -671,12 +665,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
if (!UserGroupInformation.isSecurityEnabled()) {
String userName = UserGroupInformation.getCurrentUser().getUserName();
- LOG.info("Run as user " + userName);
+ LOG.debug("Run as user " + userName);
// HADOOP_USER_NAME env is used by UserGroupInformation when log in
// This env makes AM run as this user
env.put("HADOOP_USER_NAME", userName);
}
- LOG.info("AM env: \n{}", stringifyMap(env));
+ LOG.debug("AM env: \n{}", stringifyMap(env));
return env;
}
@@ -689,11 +683,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+ fs.submitTarGzipAndUpdate(localResources);
} else {
- String[] libs = SliderUtils.getLibDirs();
+ String[] libs = ServiceUtils.getLibDirs();
+ LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
+ " apps, pre-upload dependency jars to HDFS "
+ + "using command: yarn app -enableFastLaunch");
for (String libDirProp : libs) {
ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
libDirProp);
@@ -740,7 +737,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
ApplicationId appId = submitApp(service);
service.setId(appId.toString());
// write app definition on to hdfs
- createDirAndPersistApp(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
return 0;
}
@@ -763,16 +761,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
}
- private void persistAppDef(Path appDir, Service service)
- throws IOException {
+ private Path persistAppDef(Path appDir, Service service) throws IOException {
Path appJson = new Path(appDir, service.getName() + ".json");
- jsonSerDeser
- .save(fs.getFileSystem(), appJson, service, true);
- LOG.info(
- "Persisted service " + service.getName() + " at " + appJson);
+ jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
+ return appJson;
}
private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
@@ -922,7 +918,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return EXIT_SUCCESS;
}
- String[] libDirs = SliderUtils.getLibDirs();
+ String[] libDirs = ServiceUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 7667a53..7208f39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
import org.apache.hadoop.yarn.service.monitor.probe.Probe;
import org.slf4j.Logger;
@@ -419,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
public boolean areDependenciesReady() {
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return true;
}
for (String dependency : dependencies) {
@@ -445,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> {
public Map<String, String> getDependencyHostIpTokens() {
Map<String, String> tokens = new HashMap<>();
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return tokens;
}
for (String dependency : dependencies) {
@@ -455,8 +455,8 @@ public class Component implements EventHandler<ComponentEvent> {
if (instance.getContainerStatus() == null) {
continue;
}
- if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
- SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+ if (ServiceUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+ ServiceUtils.isUnset(instance.getContainerStatus().getHost())) {
continue;
}
String ip = instance.getContainerStatus().getIPs().get(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 2492454..9e5f98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
@@ -503,7 +503,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ nodeId + ", will try again", e);
return;
}
- if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+ if (ServiceUtils.isEmpty(status.getIPs()) || ServiceUtils
.isUnset(status.getHost())) {
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index e4eae20..2d7c3bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,7 +132,7 @@ public class AbstractLauncher {
*/
public ContainerLaunchContext completeContainerLaunch() throws IOException {
- String cmdStr = SliderUtils.join(commands, " ", false);
+ String cmdStr = ServiceUtils.join(commands, " ", false);
log.debug("Completed setting up container command {}", cmdStr);
containerLaunchContext.setCommands(commands);
@@ -205,7 +205,7 @@ public class AbstractLauncher {
String key = entry.getKey();
LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ log.debug(key + "=" + ServiceUtils.stringify(val.getResource()));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
index 22b3877..711abb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -63,7 +63,7 @@ public class ClasspathConstructor {
}
public String buildClasspath() {
- return SliderUtils.join(pathElements,
+ return ServiceUtils.join(pathElements,
CLASS_PATH_SEPARATOR,
false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
index 7baa284..5ed56e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.List;
@@ -81,6 +81,6 @@ public class CommandLineBuilder {
* @return the command line
*/
public String build() {
- return SliderUtils.join(argumentList, " ");
+ return ServiceUtils.join(argumentList, " ");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index cbcb0d6..7f6cc04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import java.util.Map;
@@ -53,7 +53,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
* trimmed.
*/
public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
+ if (ServiceUtils.isSet(heap)) {
add("-Xmx" + heap.trim());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
index 1923086..1ed13a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -21,7 +21,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,7 +75,7 @@ public class HttpProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ if (containerStatus == null || ServiceUtils.isEmpty(containerStatus.getIPs())
|| StringUtils.isEmpty(containerStatus.getHost())) {
status.fail(this, new IOException("IP is not available yet"));
return status;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
index aba5859..85569f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.service.monitor.probe;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +67,7 @@ public class PortProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
- if (instance.getContainerStatus() == null || SliderUtils
+ if (instance.getContainerStatus() == null || ServiceUtils
.isEmpty(instance.getContainerStatus().getIPs())) {
status.fail(this, new IOException(
instance.getCompInstanceName() + ": IP is not available yet"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 75c647b..fc8953c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.IOException;
import java.nio.file.Paths;
@@ -53,12 +53,12 @@ public abstract class AbstractClientProvider {
public static final Set<String> createApplicationTags(String appName,
String appVersion, String appDescription) {
Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
+ tags.add(ServiceUtils.createNameTag(appName));
if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
+ tags.add(ServiceUtils.createVersionTag(appVersion));
}
if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
+ tags.add(ServiceUtils.createDescriptionTag(appDescription));
}
return tags;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6ffb84d..6d74061 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
@@ -70,7 +70,7 @@ public abstract class AbstractProviderService implements ProviderService,
.initCompTokensForSubstitute(instance);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
- launcher.putEnv(SliderUtils
+ launcher.putEnv(ServiceUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 582c5ac..e074dd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -94,7 +94,7 @@ public class ProviderUtils implements YarnServiceConstants {
IOException,
SliderException {
try {
- SliderUtils.putJar(providerResources,
+ ServiceUtils.putJar(providerResources,
sliderFileSystem,
providerClass,
tempPath,
@@ -127,14 +127,14 @@ public class ProviderUtils implements YarnServiceConstants {
String libDir,
String libLocalSrcDir)
throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
+ if (ServiceUtils.isSet(libLocalSrcDir)) {
File file = new File(libLocalSrcDir);
if (!file.exists() || !file.isDirectory()) {
throw new BadCommandArgumentsException(
"Supplied lib src dir %s is not valid", libLocalSrcDir);
}
}
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ ServiceUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
libDir, libLocalSrcDir);
}
@@ -174,7 +174,7 @@ public class ProviderUtils implements YarnServiceConstants {
Configuration conf = service.getConfiguration();
String keytabPathOnHost =
conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
+ if (ServiceUtils.isUnset(keytabPathOnHost)) {
String amKeytabName =
conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index d418b59..cecca5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,9 +70,9 @@ public class YarnRegistryViewForProviders {
Preconditions.checkArgument(registryOperations != null,
"null registry operations");
Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(serviceClass),
+ Preconditions.checkArgument(ServiceUtils.isSet(serviceClass),
"unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ Preconditions.checkArgument(ServiceUtils.isSet(instanceName),
"instanceName");
Preconditions.checkArgument(applicationAttemptId != null,
"null applicationAttemptId");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 281e1dfe..6b9d4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -409,7 +409,7 @@ public class CoreFileSystem {
*/
public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ File localFile = ServiceUtils.findContainingJarOrFail(clazz);
return submitFile(localFile, tempPath, subdir, jarName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
index 2dbf37f..1d64ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -85,7 +85,7 @@ public class PortScanner {
if (remainingPortsToCheck != null) {
return getAvailablePortViaPortArray();
} else {
- return SliderUtils.getOpenPort();
+ return ServiceUtils.getOpenPort();
}
}
@@ -95,7 +95,7 @@ public class PortScanner {
Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
while (portsToCheck.hasNext() && !found) {
int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
+ found = ServiceUtils.isPortAvailable(portToCheck);
if (found) {
availablePort = portToCheck;
portsToCheck.remove();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 68db0bb..b58cea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -369,7 +369,7 @@ public class ServiceApiUtil {
continue;
}
boolean dependenciesAlreadySorted = true;
- if (!SliderUtils.isEmpty(component.getDependencies())) {
+ if (!ServiceUtils.isEmpty(component.getDependencies())) {
for (String dependency : component.getDependencies()) {
if (!sortedComponents.containsKey(dependency)) {
dependenciesAlreadySorted = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
new file mode 100644
index 0000000..e18bcae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class ServiceUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(ServiceUtils.class);
+
+ private ServiceUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = ServiceUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.debug("Loading all dependencies from {}", srcPath);
+ if (ServiceUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 2809dfb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,564 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- @SuppressWarnings("deprecation")
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.hadoop.yarn.service.api.records.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd " +
- "enableFastLaunch");
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
index 1fa07ce..c6e8525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -32,7 +32,7 @@ public class ZookeeperUtils {
String zkPort = Integer.toString(port);
//parse the hosts
String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ String quorum = ServiceUtils.join(hostlist, ":" + zkPort + ",", false);
return quorum;
}
@@ -117,7 +117,7 @@ public class ZookeeperUtils {
for (HostAndPort hostAndPort : hostAndPorts) {
entries.add(buildQuorumEntry(hostAndPort, defaultPort));
}
- return SliderUtils.join(entries, ",", false);
+ return ServiceUtils.join(entries, ",", false);
}
public static String convertToHostsOnlyList(String quorum) throws
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
index 6159215..d7fa9a04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.service.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.junit.Assert;
import org.junit.Test;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-14872. CryptoInputStream
should implement unbuffer. Contributed by John Zhuge.
Posted by su...@apache.org.
HADOOP-14872. CryptoInputStream should implement unbuffer. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c32ddad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c32ddad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c32ddad
Branch: refs/heads/YARN-5881
Commit: 6c32ddad30240a251caaefdf7fec9ff8ad177a7c
Parents: bf6a660
Author: John Zhuge <jz...@apache.org>
Authored: Tue Nov 7 00:09:34 2017 -0800
Committer: John Zhuge <jz...@apache.org>
Committed: Thu Nov 9 10:16:12 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/crypto/CryptoInputStream.java | 32 ++++++++-
.../hadoop/crypto/CryptoStreamsTestBase.java | 72 +++++++++++++++++++-
.../apache/hadoop/crypto/TestCryptoStreams.java | 28 ++++++--
.../crypto/TestCryptoStreamsForLocalFS.java | 5 ++
.../hadoop/crypto/TestCryptoStreamsNormal.java | 5 ++
5 files changed, 133 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c32ddad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index 0be6e34..a2273bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -30,20 +30,23 @@ import java.util.EnumSet;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.CanSetDropBehind;
import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
import org.apache.hadoop.fs.HasFileDescriptor;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.StreamCapabilitiesPolicy;
import org.apache.hadoop.io.ByteBufferPool;
-
-import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.StringUtils;
/**
* CryptoInputStream decrypts data. It is not thread-safe. AES CTR mode is
@@ -61,7 +64,7 @@ import com.google.common.base.Preconditions;
public class CryptoInputStream extends FilterInputStream implements
Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor,
CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess,
- ReadableByteChannel {
+ ReadableByteChannel, CanUnbuffer, StreamCapabilities {
private final byte[] oneByteBuf = new byte[1];
private final CryptoCodec codec;
private final Decryptor decryptor;
@@ -719,4 +722,27 @@ public class CryptoInputStream extends FilterInputStream implements
public boolean isOpen() {
return !closed;
}
+
+ private void cleanDecryptorPool() {
+ decryptorPool.clear();
+ }
+
+ @Override
+ public void unbuffer() {
+ cleanBufferPool();
+ cleanDecryptorPool();
+ StreamCapabilitiesPolicy.unbuffer(in);
+ }
+
+ @Override
+ public boolean hasCapability(String capability) {
+ switch (StringUtils.toLowerCase(capability)) {
+ case StreamCapabilities.READAHEAD:
+ case StreamCapabilities.DROPBEHIND:
+ case StreamCapabilities.UNBUFFER:
+ return true;
+ default:
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c32ddad/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
index 9183524..259383d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
@@ -27,6 +27,7 @@ import java.util.EnumSet;
import java.util.Random;
import org.apache.hadoop.fs.ByteBufferReadable;
+import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
@@ -102,7 +103,32 @@ public abstract class CryptoStreamsTestBase {
return total;
}
-
+
+ private int preadAll(PositionedReadable in, byte[] b, int off, int len)
+ throws IOException {
+ int n = 0;
+ int total = 0;
+ while (n != -1) {
+ total += n;
+ if (total >= len) {
+ break;
+ }
+ n = in.read(total, b, off + total, len - total);
+ }
+
+ return total;
+ }
+
+ private void preadCheck(PositionedReadable in) throws Exception {
+ byte[] result = new byte[dataLen];
+ int n = preadAll(in, result, 0, dataLen);
+
+ Assert.assertEquals(dataLen, n);
+ byte[] expectedData = new byte[n];
+ System.arraycopy(data, 0, expectedData, 0, n);
+ Assert.assertArrayEquals(result, expectedData);
+ }
+
protected OutputStream getOutputStream(int bufferSize) throws IOException {
return getOutputStream(bufferSize, key, iv);
}
@@ -146,7 +172,6 @@ public abstract class CryptoStreamsTestBase {
// EOF
n = in.read(result, 0, dataLen);
Assert.assertEquals(n, -1);
- in.close();
}
/** Test crypto writing with different buffer size. */
@@ -730,4 +755,47 @@ public abstract class CryptoStreamsTestBase {
in.close();
}
+
+ /** Test unbuffer. */
+ @Test(timeout=120000)
+ public void testUnbuffer() throws Exception {
+ OutputStream out = getOutputStream(smallBufferSize);
+ writeData(out);
+
+ // Test buffered read
+ try (InputStream in = getInputStream(smallBufferSize)) {
+ // Test unbuffer after buffered read
+ readCheck(in);
+ ((CanUnbuffer) in).unbuffer();
+
+ if (in instanceof Seekable) {
+ // Test buffered read again after unbuffer
+ // Must seek to the beginning first
+ ((Seekable) in).seek(0);
+ readCheck(in);
+ }
+
+ // Test close after unbuffer
+ ((CanUnbuffer) in).unbuffer();
+ // The close will be called when exiting this try-with-resource block
+ }
+
+ // Test pread
+ try (InputStream in = getInputStream(smallBufferSize)) {
+ if (in instanceof PositionedReadable) {
+ PositionedReadable pin = (PositionedReadable) in;
+
+ // Test unbuffer after pread
+ preadCheck(pin);
+ ((CanUnbuffer) in).unbuffer();
+
+ // Test pread again after unbuffer
+ preadCheck(pin);
+
+ // Test close after unbuffer
+ ((CanUnbuffer) in).unbuffer();
+ // The close will be called when exiting this try-with-resource block
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c32ddad/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java
index a14f9a3..027ac93 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreams.java
@@ -29,11 +29,13 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.CanSetDropBehind;
import org.apache.hadoop.fs.CanSetReadahead;
+import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
import org.apache.hadoop.fs.HasFileDescriptor;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.io.ByteBufferPool;
import org.apache.hadoop.io.DataInputBuffer;
@@ -159,16 +161,18 @@ public class TestCryptoStreams extends CryptoStreamsTestBase {
}
}
- public static class FakeInputStream extends InputStream implements
- Seekable, PositionedReadable, ByteBufferReadable, HasFileDescriptor,
- CanSetDropBehind, CanSetReadahead, HasEnhancedByteBufferAccess {
+ static class FakeInputStream extends InputStream
+ implements Seekable, PositionedReadable, ByteBufferReadable,
+ HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
+ HasEnhancedByteBufferAccess, CanUnbuffer,
+ StreamCapabilities {
private final byte[] oneByteBuf = new byte[1];
private int pos = 0;
private final byte[] data;
private final int length;
private boolean closed = false;
- public FakeInputStream(DataInputBuffer in) {
+ FakeInputStream(DataInputBuffer in) {
data = in.getData();
length = in.getLength();
}
@@ -350,6 +354,22 @@ public class TestCryptoStreams extends CryptoStreamsTestBase {
}
@Override
+ public void unbuffer() {
+ }
+
+ @Override
+ public boolean hasCapability(String capability) {
+ switch (capability.toLowerCase()) {
+ case StreamCapabilities.READAHEAD:
+ case StreamCapabilities.DROPBEHIND:
+ case StreamCapabilities.UNBUFFER:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @Override
public FileDescriptor getFileDescriptor() throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c32ddad/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
index 1ef6f3c..bb3fd7a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsForLocalFS.java
@@ -112,4 +112,9 @@ public class TestCryptoStreamsForLocalFS extends CryptoStreamsTestBase {
@Test(timeout=10000)
public void testSeekToNewSource() throws Exception {
}
+
+ @Ignore("Local file input stream does not support unbuffer")
+ @Override
+ @Test
+ public void testUnbuffer() throws Exception {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c32ddad/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
index b5382c1..7e30077 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsNormal.java
@@ -120,4 +120,9 @@ public class TestCryptoStreamsNormal extends CryptoStreamsTestBase {
@Override
@Test(timeout=10000)
public void testHasEnhancedByteBufferAccess() throws IOException {}
+
+ @Ignore("ByteArrayInputStream does not support unbuffer")
+ @Override
+ @Test
+ public void testUnbuffer() throws Exception {}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-14960. Add GC time
percentage monitor/alerter. Contributed by Misha Dmitriev.
Posted by su...@apache.org.
HADOOP-14960. Add GC time percentage monitor/alerter. Contributed by Misha Dmitriev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c6adda2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c6adda2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c6adda2
Branch: refs/heads/YARN-5881
Commit: 3c6adda291745c592741b87cd613214ae11887e4
Parents: 10a1f55
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Nov 9 21:05:34 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Nov 9 21:06:06 2017 -0800
----------------------------------------------------------------------
.../hadoop/metrics2/source/JvmMetrics.java | 15 ++
.../hadoop/metrics2/source/JvmMetricsInfo.java | 3 +-
.../org/apache/hadoop/util/GcTimeMonitor.java | 242 +++++++++++++++++++
.../hadoop/metrics2/source/TestJvmMetrics.java | 92 ++++++-
4 files changed, 345 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c6adda2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index e3f8754..8c3375f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -28,6 +28,8 @@ import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.log.metrics.EventCounter;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -39,6 +41,8 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.Interns;
import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
import static org.apache.hadoop.metrics2.impl.MsInfo.*;
+
+import org.apache.hadoop.util.GcTimeMonitor;
import org.apache.hadoop.util.JvmPauseMonitor;
/**
@@ -85,6 +89,7 @@ public class JvmMetrics implements MetricsSource {
private JvmPauseMonitor pauseMonitor = null;
final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
new ConcurrentHashMap<String, MetricsInfo[]>();
+ private GcTimeMonitor gcTimeMonitor = null;
@VisibleForTesting
JvmMetrics(String processName, String sessionId) {
@@ -96,6 +101,11 @@ public class JvmMetrics implements MetricsSource {
this.pauseMonitor = pauseMonitor;
}
+ public void setGcTimeMonitor(GcTimeMonitor gcTimeMonitor) {
+ Preconditions.checkNotNull(gcTimeMonitor);
+ this.gcTimeMonitor = gcTimeMonitor;
+ }
+
public static JvmMetrics create(String processName, String sessionId,
MetricsSystem ms) {
return ms.register(JvmMetrics.name(), JvmMetrics.description(),
@@ -176,6 +186,11 @@ public class JvmMetrics implements MetricsSource {
rb.addCounter(GcTotalExtraSleepTime,
pauseMonitor.getTotalGcExtraSleepTime());
}
+
+ if (gcTimeMonitor != null) {
+ rb.addCounter(GcTimePercentage,
+ gcTimeMonitor.getLatestGcData().getGcTimePercentage());
+ }
}
private MetricsInfo[] getGcInfo(String gcName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c6adda2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
index 8da6785..74f670b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
@@ -51,7 +51,8 @@ public enum JvmMetricsInfo implements MetricsInfo {
LogInfo("Total number of info log events"),
GcNumWarnThresholdExceeded("Number of times that the GC warn threshold is exceeded"),
GcNumInfoThresholdExceeded("Number of times that the GC info threshold is exceeded"),
- GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds");
+ GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds"),
+ GcTimePercentage("Percentage of time the JVM was paused in GC");
private final String desc;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c6adda2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
new file mode 100644
index 0000000..0640fc0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import com.google.common.base.Preconditions;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+/**
+ * This class monitors the percentage of time the JVM is paused in GC within
+ * the specified observation window, say 1 minute. The user can provide a
+ * hook which will be called whenever this percentage exceeds the specified
+ * threshold.
+ */
+public class GcTimeMonitor extends Thread {
+
+ private final long maxGcTimePercentage;
+ private final long observationWindowMs, sleepIntervalMs;
+ private final GcTimeAlertHandler alertHandler;
+
+ private final List<GarbageCollectorMXBean> gcBeans =
+ ManagementFactory.getGarbageCollectorMXBeans();
+ // Ring buffers containing GC timings and timestamps when timings were taken
+ private final TsAndData[] gcDataBuf;
+ private int bufSize, startIdx, endIdx;
+
+ private long startTime;
+ private final GcData curData = new GcData();
+ private volatile boolean shouldRun = true;
+
+ /**
+ * Create an instance of GCTimeMonitor. Once it's started, it will stay alive
+ * and monitor GC time percentage until shutdown() is called. If you don't
+ * put a limit on the number of GCTimeMonitor instances that you create, and
+ * alertHandler != null, you should necessarily call shutdown() once the given
+ * instance is not needed. Otherwise, you may create a memory leak, because
+ * each running GCTimeMonitor will keep its alertHandler object in memory,
+ * which in turn may reference and keep in memory many more other objects.
+ *
+ * @param observationWindowMs the interval over which the percentage
+ * of GC time should be calculated. A practical value would be somewhere
+ * between 30 sec and several minutes.
+ * @param sleepIntervalMs how frequently this thread should wake up to check
+ * GC timings. This is also a frequency with which alertHandler will be
+ * invoked if GC time percentage exceeds the specified limit. A practical
+ * value would likely be 500..1000 ms.
+ * @param maxGcTimePercentage A GC time percentage limit (0..100) within
+ * observationWindowMs. Once this is exceeded, alertHandler will be
+ * invoked every sleepIntervalMs milliseconds until GC time percentage
+ * falls below this limit.
+ * @param alertHandler a single method in this interface is invoked when GC
+ * time percentage exceeds the specified limit.
+ */
+ public GcTimeMonitor(long observationWindowMs, long sleepIntervalMs,
+ int maxGcTimePercentage, GcTimeAlertHandler alertHandler) {
+ Preconditions.checkArgument(observationWindowMs > 0);
+ Preconditions.checkArgument(
+ sleepIntervalMs > 0 && sleepIntervalMs < observationWindowMs);
+ Preconditions.checkArgument(
+ maxGcTimePercentage >= 0 && maxGcTimePercentage <= 100);
+
+ this.observationWindowMs = observationWindowMs;
+ this.sleepIntervalMs = sleepIntervalMs;
+ this.maxGcTimePercentage = maxGcTimePercentage;
+ this.alertHandler = alertHandler;
+
+ bufSize = (int) (observationWindowMs / sleepIntervalMs + 2);
+ // Prevent the user from accidentally creating an abnormally big buffer,
+ // which will result in slow calculations and likely inaccuracy.
+ Preconditions.checkArgument(bufSize <= 128 * 1024);
+ gcDataBuf = new TsAndData[bufSize];
+ for (int i = 0; i < bufSize; i++) {
+ gcDataBuf[i] = new TsAndData();
+ }
+
+ this.setDaemon(true);
+ this.setName("GcTimeMonitor obsWindow = " + observationWindowMs +
+ ", sleepInterval = " + sleepIntervalMs +
+ ", maxGcTimePerc = " + maxGcTimePercentage);
+ }
+
+ @Override
+ public void run() {
+ startTime = System.currentTimeMillis();
+ curData.timestamp = startTime;
+ gcDataBuf[startIdx].setValues(startTime, 0);
+
+ while (shouldRun) {
+ try {
+ Thread.sleep(sleepIntervalMs);
+ } catch (InterruptedException ie) {
+ return;
+ }
+
+ calculateGCTimePercentageWithinObservedInterval();
+ if (alertHandler != null &&
+ curData.gcTimePercentage > maxGcTimePercentage) {
+ alertHandler.alert(curData.clone());
+ }
+ }
+ }
+
+ public void shutdown() {
+ shouldRun = false;
+ }
+
+ /** Returns a copy of the most recent data measured by this monitor. */
+ public GcData getLatestGcData() {
+ return curData.clone();
+ }
+
+ private void calculateGCTimePercentageWithinObservedInterval() {
+ long prevTotalGcTime = curData.totalGcTime;
+ long totalGcTime = 0;
+ long totalGcCount = 0;
+ for (GarbageCollectorMXBean gcBean : gcBeans) {
+ totalGcTime += gcBean.getCollectionTime();
+ totalGcCount += gcBean.getCollectionCount();
+ }
+ long gcTimeWithinSleepInterval = totalGcTime - prevTotalGcTime;
+
+ long ts = System.currentTimeMillis();
+ long gcMonitorRunTime = ts - startTime;
+
+ endIdx = (endIdx + 1) % bufSize;
+ gcDataBuf[endIdx].setValues(ts, gcTimeWithinSleepInterval);
+
+ // Move startIdx forward until we reach the first buffer entry with
+ // timestamp within the observation window.
+ long startObsWindowTs = ts - observationWindowMs;
+ while (gcDataBuf[startIdx].ts < startObsWindowTs && startIdx != endIdx) {
+ startIdx = (startIdx + 1) % bufSize;
+ }
+
+ // Calculate total GC time within observationWindowMs.
+ // We should be careful about GC time that passed before the first timestamp
+ // in our observation window.
+ long gcTimeWithinObservationWindow = Math.min(
+ gcDataBuf[startIdx].gcPause, gcDataBuf[startIdx].ts - startObsWindowTs);
+ if (startIdx != endIdx) {
+ for (int i = (startIdx + 1) % bufSize; i != endIdx;
+ i = (i + 1) % bufSize) {
+ gcTimeWithinObservationWindow += gcDataBuf[i].gcPause;
+ }
+ }
+
+ curData.update(ts, gcMonitorRunTime, totalGcTime, totalGcCount,
+ (int) (gcTimeWithinObservationWindow * 100 /
+ Math.min(observationWindowMs, gcMonitorRunTime)));
+ }
+
+ /**
+ * The user can provide an instance of a class implementing this interface
+ * when initializing a GcTimeMonitor to receive alerts when GC time
+ * percentage exceeds the specified threshold.
+ */
+ public interface GcTimeAlertHandler {
+ void alert(GcData gcData);
+ }
+
+ /** Encapsulates data about GC pauses measured at the specific timestamp. */
+ public static class GcData implements Cloneable {
+ private long timestamp;
+ private long gcMonitorRunTime, totalGcTime, totalGcCount;
+ private int gcTimePercentage;
+
+ /** Returns the absolute timestamp when this measurement was taken. */
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ /** Returns the time since the start of the associated GcTimeMonitor. */
+ public long getGcMonitorRunTime() {
+ return gcMonitorRunTime;
+ }
+
+ /** Returns accumulated GC time since this JVM started. */
+ public long getAccumulatedGcTime() {
+ return totalGcTime;
+ }
+
+ /** Returns the accumulated number of GC pauses since this JVM started. */
+ public long getAccumulatedGcCount() {
+ return totalGcCount;
+ }
+
+ /**
+ * Returns the percentage (0..100) of time that the JVM spent in GC pauses
+ * within the observation window of the associated GcTimeMonitor.
+ */
+ public int getGcTimePercentage() {
+ return gcTimePercentage;
+ }
+
+ private synchronized void update(long inTimestamp, long inGcMonitorRunTime,
+ long inTotalGcTime, long inTotalGcCount, int inGcTimePercentage) {
+ this.timestamp = inTimestamp;
+ this.gcMonitorRunTime = inGcMonitorRunTime;
+ this.totalGcTime = inTotalGcTime;
+ this.totalGcCount = inTotalGcCount;
+ this.gcTimePercentage = inGcTimePercentage;
+ }
+
+ @Override
+ public synchronized GcData clone() {
+ try {
+ return (GcData) super.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private static class TsAndData {
+ private long ts; // Timestamp when this measurement was taken
+ private long gcPause; // Total GC pause time within the interval between ts
+ // and the timestamp of the previous measurement.
+
+ void setValues(long inTs, long inGcPause) {
+ this.ts = inTs;
+ this.gcPause = inGcPause;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c6adda2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
index 69e7565..5320b6e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.metrics2.source;
+import org.apache.hadoop.util.GcTimeMonitor;
import org.junit.After;
import org.junit.Assert;
import org.junit.Rule;
@@ -35,6 +36,9 @@ import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.JvmPauseMonitor;
+import java.util.ArrayList;
+import java.util.List;
+
import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
import static org.apache.hadoop.metrics2.impl.MsInfo.*;
@@ -43,17 +47,21 @@ public class TestJvmMetrics {
@Rule
public Timeout timeout = new Timeout(30000);
private JvmPauseMonitor pauseMonitor;
+ private GcTimeMonitor gcTimeMonitor;
/**
- * Robust shutdown of the pause monitor if it hasn't been stopped already.
+ * Robust shutdown of the monitors if they haven't been stopped already.
*/
@After
public void teardown() {
ServiceOperations.stop(pauseMonitor);
+ if (gcTimeMonitor != null) {
+ gcTimeMonitor.shutdown();
+ }
}
@Test
- public void testPresence() {
+ public void testJvmPauseMonitorPresence() {
pauseMonitor = new JvmPauseMonitor();
pauseMonitor.init(new Configuration());
pauseMonitor.start();
@@ -66,14 +74,35 @@ public class TestJvmMetrics {
verify(rb).tag(ProcessName, "test");
verify(rb).tag(SessionId, "test");
for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
- if (info.name().startsWith("Mem"))
+ if (info.name().startsWith("Mem")) {
verify(rb).addGauge(eq(info), anyFloat());
- else if (info.name().startsWith("Gc"))
+ } else if (info.name().startsWith("Gc") &&
+ !info.name().equals("GcTimePercentage")) {
verify(rb).addCounter(eq(info), anyLong());
- else if (info.name().startsWith("Threads"))
+ } else if (info.name().startsWith("Threads")) {
verify(rb).addGauge(eq(info), anyInt());
- else if (info.name().startsWith("Log"))
+ } else if (info.name().startsWith("Log")) {
verify(rb).addCounter(eq(info), anyLong());
+ }
+ }
+ }
+
+ @Test
+ public void testGcTimeMonitorPresence() {
+ gcTimeMonitor = new GcTimeMonitor(60000, 1000, 70, null);
+ gcTimeMonitor.start();
+ JvmMetrics jvmMetrics = new JvmMetrics("test", "test");
+ jvmMetrics.setGcTimeMonitor(gcTimeMonitor);
+ MetricsRecordBuilder rb = getMetrics(jvmMetrics);
+ MetricsCollector mc = rb.parent();
+
+ verify(mc).addRecord(JvmMetrics);
+ verify(rb).tag(ProcessName, "test");
+ verify(rb).tag(SessionId, "test");
+ for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
+ if (info.name().equals("GcTimePercentage")) {
+ verify(rb).addCounter(eq(info), anyInt());
+ }
}
}
@@ -120,4 +149,55 @@ public class TestJvmMetrics {
}
}
+ @Test
+ public void testGcTimeMonitor() {
+ class Alerter implements GcTimeMonitor.GcTimeAlertHandler {
+ private volatile int numAlerts;
+ private volatile int maxGcTimePercentage;
+ @Override
+ public void alert(GcTimeMonitor.GcData gcData) {
+ numAlerts++;
+ if (gcData.getGcTimePercentage() > maxGcTimePercentage) {
+ maxGcTimePercentage = gcData.getGcTimePercentage();
+ }
+ }
+ }
+ Alerter alerter = new Alerter();
+
+ int alertGcPerc = 10; // Alerter should be called if GC takes >= 10%
+ gcTimeMonitor = new GcTimeMonitor(60*1000, 100, alertGcPerc, alerter);
+ gcTimeMonitor.start();
+
+ int maxGcTimePercentage = 0;
+ long gcCount = 0;
+
+ // Generate a lot of garbage for some time and verify that the monitor
+ // reports at least some percentage of time in GC pauses, and that the
+ // alerter is invoked at least once.
+
+ List<String> garbageStrings = new ArrayList<>();
+
+ long startTime = System.currentTimeMillis();
+ // Run this for at least 1 sec for our monitor to collect enough data
+ while (System.currentTimeMillis() - startTime < 1000) {
+ for (int j = 0; j < 100000; j++) {
+ garbageStrings.add(
+ "Long string prefix just to fill memory with garbage " + j);
+ }
+ garbageStrings.clear();
+ System.gc();
+
+ GcTimeMonitor.GcData gcData = gcTimeMonitor.getLatestGcData();
+ int gcTimePercentage = gcData.getGcTimePercentage();
+ if (gcTimePercentage > maxGcTimePercentage) {
+ maxGcTimePercentage = gcTimePercentage;
+ }
+ gcCount = gcData.getAccumulatedGcCount();
+ }
+
+ Assert.assertTrue(maxGcTimePercentage > 0);
+ Assert.assertTrue(gcCount > 0);
+ Assert.assertTrue(alerter.numAlerts > 0);
+ Assert.assertTrue(alerter.maxGcTimePercentage >= alertGcPerc);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-7393. RegistryDNS doesn't
work in tcp channel. Contributed by Eric Yang
Posted by su...@apache.org.
YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ec9e72c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ec9e72c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ec9e72c
Branch: refs/heads/YARN-5881
Commit: 8ec9e72cc37957d8cf9af5a7b78086441ae64b00
Parents: d2775fb
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Oct 27 11:44:55 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../hadoop/registry/server/dns/RegistryDNS.java | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ec9e72c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 9097c46..d7ebece 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,8 +189,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
addNIOUDP(addr, port);
- //TODO Fix dns lookup over TCP
-// addNIOTCP(addr, port);
+ addNIOTCP(addr, port);
}
/**
@@ -768,7 +767,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
buf.get(in, 0, messageLength);
Message query;
- byte[] response = null;
+ byte[] response;
try {
query = new Message(in);
LOG.info("received TCP query {}", query.getQuestion());
@@ -781,10 +780,19 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
ByteBuffer out = ByteBuffer.allocate(response.length + 2);
- out.putShort(0, (short) (response.length & 0xffff));
+ out.clear();
+ byte[] data = new byte[2];
+
+ data[1] = (byte)(response.length & 0xFF);
+ data[0] = (byte)((response.length >> 8) & 0xFF);
+ out.put(data);
out.put(response);
+ out.flip();
+
+ while(out.hasRemaining()) {
+ ch.write(out);
+ }
- ch.write(out);
} catch (IOException e) {
throw NetUtils.wrapException(ch.socket().getInetAddress().getHostName(),
ch.socket().getPort(),
@@ -836,6 +844,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
});
+ } else {
+ Thread.sleep(500);
}
}
} catch (IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 0544387..4985a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collections;
@@ -121,6 +123,15 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
// For queue with same used ratio / priority, queue with higher configured
// capacity goes first
if (0 == rc) {
+ Resource minEffRes1 = q1.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ Resource minEffRes2 = q2.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ if (!minEffRes1.equals(Resources.none())
+ && !minEffRes2.equals(Resources.none())) {
+ return minEffRes2.compareTo(minEffRes1);
+ }
+
float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
return Float.compare(abs2, abs1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 22705cc..86b2fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -62,6 +62,8 @@ public class CapacitySchedulerQueueInfo {
protected long pendingContainers;
protected QueueCapacitiesInfo capacities;
protected ResourcesInfo resources;
+ protected ResourceInfo minEffectiveCapacity;
+ protected ResourceInfo maxEffectiveCapacity;
CapacitySchedulerQueueInfo() {
};
@@ -105,6 +107,11 @@ public class CapacitySchedulerQueueInfo {
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
+
+ minEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMinResource());
+ maxEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMaxResource());
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -200,4 +207,12 @@ public class CapacitySchedulerQueueInfo {
public ResourcesInfo getResources() {
return resources;
}
+
+ public ResourceInfo getMinEffectiveCapacity(){
+ return minEffectiveCapacity;
+ }
+
+ public ResourceInfo getMaxEffectiveCapacity(){
+ return maxEffectiveCapacity;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index b5c20f4..909d023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -142,6 +143,13 @@ public class MockNM {
return this.registeringCollectors;
}
+ public void unRegisterNode() throws Exception {
+ UnRegisterNodeManagerRequest request = Records
+ .newRecord(UnRegisterNodeManagerRequest.class);
+ request.setNodeId(nodeId);
+ resourceTracker.unRegisterNodeManager(request);
+ }
+
public RegisterNodeManagerResponse registerNode() throws Exception {
return registerNode(null, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index d128b02..d982d2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -822,6 +822,12 @@ public class MockRM extends ResourceManager {
return rmApp;
}
+ public MockNM unRegisterNode(MockNM nm) throws Exception {
+ nm.unRegisterNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
nm.registerNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 4fc0ea4..591d5f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
@@ -641,9 +642,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
QueueCapacities qc = new QueueCapacities(0 == myLevel);
ResourceUsage ru = new ResourceUsage();
+ QueueResourceQuotas qr = new QueueResourceQuotas();
when(queue.getQueueCapacities()).thenReturn(qc);
when(queue.getQueueResourceUsage()).thenReturn(ru);
+ when(queue.getQueueResourceQuotas()).thenReturn(qr);
LOG.debug("Setup queue, name=" + queue.getQueueName() + " path="
+ queue.getQueuePath());
@@ -676,7 +679,17 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
qc.setAbsoluteMaximumCapacity(partitionName, absMax);
qc.setAbsoluteUsedCapacity(partitionName, absUsed);
qc.setUsedCapacity(partitionName, used);
+ qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(parseResourceFromString(values[0].trim()));
+ qr.setEffectiveMaxResource(partitionName,
+ parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(partitionName,
+ parseResourceFromString(values[0].trim()));
when(queue.getUsedCapacity()).thenReturn(used);
+ when(queue.getEffectiveCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[0].trim()));
+ when(queue.getEffectiveMaxCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[1].trim()));
ru.setPending(partitionName, pending);
// Setup reserved resource if it contained by input config
Resource reserved = Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 694be09..dd950f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -437,8 +437,8 @@ public class TestProportionalCapacityPreemptionPolicy {
policy.editSchedule();
// queueF(appD) wants resources, Verify that resources come from queueE(appC)
// because it's a sibling and queueB(appA) because queueA is over capacity.
- verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA)));
- verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC)));
// Need to call setup() again to reset mDisp
setup();
@@ -1202,6 +1202,17 @@ public class TestProportionalCapacityPreemptionPolicy {
when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
boolean preemptionDisabled = mockPreemptionStatus("root");
when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
+ QueueResourceQuotas rootQr = new QueueResourceQuotas();
+ rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(abs[0]);
+ rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]);
+ when(root.getQueueResourceQuotas()).thenReturn(rootQr);
+ when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[0]);
+ when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[0], 0));
for (int i = 1; i < queues.length; ++i) {
final CSQueue q;
@@ -1232,6 +1243,18 @@ public class TestProportionalCapacityPreemptionPolicy {
qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
when(q.getQueueCapacities()).thenReturn(qc);
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(abs[i]);
+ qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]);
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
+ when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[i]);
+ when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[i], 0));
+
String parentPathName = p.getQueuePath();
parentPathName = (parentPathName == null) ? "root" : parentPathName;
String queuePathName = (parentPathName + "." + queueName).replace("/",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
index 7784549..a1d89d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
@@ -67,9 +67,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
"priority_first");
- String labelsConfig = "=100:200,true;";
+ String labelsConfig = "=100:50,true;";
String nodesConfig = // n1 has no label
- "n1= res=100:200";
+ "n1= res=100:50";
String queuesConfig =
// guaranteed,max,used,pending,reserved
"root(=[100:50 100:50 80:40 120:60 0]);" + // root
@@ -105,7 +105,7 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
- verify(mDisp, times(7)).handle(argThat(
+ verify(mDisp, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
new file mode 100644
index 0000000..5a66281
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -0,0 +1,516 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAbsoluteResourceConfiguration {
+
+ private static final int GB = 1024;
+
+ private static final String QUEUEA = "queueA";
+ private static final String QUEUEB = "queueB";
+ private static final String QUEUEC = "queueC";
+ private static final String QUEUEA1 = "queueA1";
+ private static final String QUEUEA2 = "queueA2";
+ private static final String QUEUEB1 = "queueB1";
+
+ private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEA;
+ private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEB;
+ private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEC;
+ private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
+ private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
+ private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+
+ private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
+ 10);
+ private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB,
+ 30);
+ private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB,
+ 10);
+ private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB,
+ 30);
+ private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB,
+ 20);
+ private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6);
+ private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000,
+ 20);
+
+ private static Set<String> resourceTypes = new HashSet<>(
+ Arrays.asList("memory", "vcores"));
+
+ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUEA1_FULL, 50f);
+ csConf.setCapacity(QUEUEA2_FULL, 50f);
+ csConf.setCapacity(QUEUEB1_FULL, 100f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ @Test
+ public void testSimpleMinMaxResourceConfigurartionPerQueue() {
+
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true);
+ setupMinMaxResourceConfiguration(csConf);
+
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ }
+
+ @Test
+ public void testEffectiveMinMaxResourceConfigurartionPerQueue()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases 1) Configure percentage based capacity and absolute
+ * resource together. 2) As per above tree structure, ensure all values
+ * could be retrieved. 3) Validate whether min resource cannot be more than
+ * max resources. 4) Validate whether max resource of queue cannot be more
+ * than its parent max resource.
+ */
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // 1. Create a new config with capcity and min/max together. Ensure an
+ // exception is thrown.
+ CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
+ true);
+ setupMinMaxResourceConfiguration(csConf1);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Queue 'queueA' should use either"
+ + " percentage based capacity configuration or absolute resource.",
+ e.getMessage());
+ }
+ rm.stop();
+
+ // 2. Create a new config with min/max alone with a complex queue config.
+ // Check all values could be fetched correctly.
+ CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf2);
+
+ rm = new MockRM(csConf2);
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+ cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+ Assert.assertNotNull(qB1);
+ Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // 3. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf3);
+
+ csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES);
+ csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ try {
+ cs.reinitialize(csConf3, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Min resource configuration "
+ + "<memory:153600, vCores:30> is greater than its "
+ + "max value:<memory:40960, vCores:10> in queue:queueB1",
+ e.getMessage());
+ }
+
+ // 4. Create a new config and make sure one queue's max resource is more
+ // than its preant's max resource configured.
+ CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf4);
+
+ csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES);
+
+ try {
+ cs.reinitialize(csConf4, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert
+ .assertEquals(
+ "Failed to re-init queues : Max resource configuration "
+ + "<memory:204800, vCores:30> is greater than parents max value:"
+ + "<memory:153600, vCores:30> in queue:queueB1",
+ e.getMessage());
+ }
+ rm.stop();
+ }
+
+ @Test
+ public void testComplexValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases: 1) Parent and its child queues must use either
+ * percentage based or absolute resource configuration. 2) Parent's min
+ * resource must be more than sum of child's min resource.
+ */
+
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // 1. Explicitly set percentage based config for parent queues. This will
+ // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
+ // resource.
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+
+ // Also unset resource based config.
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ try {
+ cs.reinitialize(csConf, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Parent queue 'queueA' "
+ + "and child queue 'queueA1' should use either percentage based"
+ + " capacity configuration or absolute resource together.",
+ e.getMessage());
+ }
+
+ // 2. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf1);
+
+ // Configure QueueA with lesser resource than its children.
+ csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: "
+ + "<memory:51200, vCores:5> is less than to its children:"
+ + "<memory:102400, vCores:10> for queue:queueA", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEffectiveResourceAfterReducingClusterResource()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+ rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // unregister one NM.
+ rm.unRegisterNode(nm1);
+
+ // After loosing one NM, effective min res of queueA will become just
+ // above half. Hence A's min will be 60Gi and 6 cores and max will be
+ // 128GB and 20 cores.
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 8aca235..24ae244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -86,6 +87,7 @@ public class TestApplicationLimits {
final static int GB = 1024;
LeafQueue queue;
+ CSQueue root;
private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -100,7 +102,7 @@ public class TestApplicationLimits {
setupQueueConfiguration(csConf);
rmContext = TestUtils.getMockRMContext();
-
+ Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
@@ -110,10 +112,11 @@ public class TestApplicationLimits {
when(csContext.getMaximumResourceCapability()).
thenReturn(Resources.createResource(16*GB, 32));
when(csContext.getClusterResource()).
- thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
+ thenReturn(clusterResource);
when(csContext.getResourceCalculator()).
thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
RMContainerTokenSecretManager containerTokenSecretManager =
new RMContainerTokenSecretManager(conf);
@@ -122,13 +125,17 @@ public class TestApplicationLimits {
containerTokenSecretManager);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
- CSQueue root = CapacitySchedulerQueueManager
+ root = CapacitySchedulerQueueManager
.parseQueue(csContext, csConf, null, "root",
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
-
queue = spy(new LeafQueue(csContext, A, root, null));
+ QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
+ .getQueueResourceQuotas();
+ doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
// Stub out ACL checks
doReturn(true).
@@ -189,6 +196,8 @@ public class TestApplicationLimits {
// when there is only 1 user, and drops to 2G (the userlimit) when there
// is a second user
Resource clusterResource = Resource.newInstance(80 * GB, 40);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
@@ -287,6 +296,8 @@ public class TestApplicationLimits {
CSQueue root =
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
"root", queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
LeafQueue queue = (LeafQueue)queues.get(A);
@@ -357,6 +368,8 @@ public class TestApplicationLimits {
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
clusterResource = Resources.createResource(100 * 16 * GB);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
@@ -378,6 +391,8 @@ public class TestApplicationLimits {
root = CapacitySchedulerQueueManager.parseQueue(
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
@@ -393,7 +408,7 @@ public class TestApplicationLimits {
final String user_0 = "user_0";
final String user_1 = "user_1";
final String user_2 = "user_2";
-
+
assertEquals(Resource.newInstance(16 * GB, 1),
queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(8 * GB, 1),
@@ -578,6 +593,7 @@ public class TestApplicationLimits {
thenReturn(Resources.createResource(16*GB));
when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
// Say cluster has 100 nodes of 16G each
Resource clusterResource = Resources.createResource(100 * 16 * GB);
@@ -586,6 +602,8 @@ public class TestApplicationLimits {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -693,6 +711,8 @@ public class TestApplicationLimits {
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.createResource(90*16*GB);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Any change is cluster resource needs to enforce user-limit recomputation.
// In existing code, LeafQueue#updateClusterResource handled this. However
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 0aac2ef..d73f1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -600,6 +601,7 @@ public class TestApplicationLimitsByPartition {
RMContext spyRMContext = spy(rmContext);
when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
when(csContext.getRMContext()).thenReturn(spyRMContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
mgr.activateNode(NodeId.newInstance("h0", 0),
Resource.newInstance(160 * GB, 16)); // default Label
@@ -615,6 +617,8 @@ public class TestApplicationLimitsByPartition {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 800d023..bbac4b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4306,7 +4306,7 @@ public class TestCapacityScheduler {
null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
- assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
+ assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
cs.getQueue("p2").getUsedResources().getMemorySize());
//Free a container from X1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 0fcc86d..fcfa0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -242,6 +242,8 @@ public class TestChildQueueOrder {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index a32352b..96f8970 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -191,6 +191,8 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
+ new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32)));
ResourceUsage queueResUsage = root.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -307,13 +309,11 @@ public class TestLeafQueue {
// Verify the value for getAMResourceLimit for queues with < .1 maxcap
Resource clusterResource = Resource.newInstance(50 * GB, 50);
- a.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(1 * GB, 1),
a.calculateAndGetAMResourceLimit());
- b.updateClusterResource(clusterResource,
- new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(5 * GB, 1),
b.calculateAndGetAMResourceLimit());
}
@@ -358,6 +358,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -556,6 +558,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -630,6 +634,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -699,6 +705,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (80 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Set user-limit. Need a small queue within a large cluster.
b.setUserLimit(50);
@@ -779,6 +787,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests so that one application is memory dominant
// and other application is vcores dominant
@@ -891,6 +901,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -915,6 +927,8 @@ public class TestLeafQueue {
// Set user-limit
a.setUserLimit(50);
a.setUserLimitFactor(2);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// There're two active users
assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers());
@@ -940,7 +954,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
// Allocate one container to app_0, before allocating this container,
- // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <=
+ // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <=
// user-limit.
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_1,
@@ -1068,15 +1082,9 @@ public class TestLeafQueue {
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(9*GB, a.getUsedResources().getMemorySize());
- assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-
- assertEquals(4*GB,
- app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
-
- assertEquals(1*GB,
- app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+ assertEquals(12*GB, a.getUsedResources().getMemorySize());
+ assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1100,6 +1108,8 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager
= mock(CapacitySchedulerQueueManager.class);
@@ -1122,6 +1132,8 @@ public class TestLeafQueue {
qb.setUserLimit(100);
qb.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
final ApplicationAttemptId appAttemptId_0 =
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
@@ -1256,106 +1268,6 @@ public class TestLeafQueue {
}
@Test
- public void testUserHeadroomMultiApp() throws Exception {
- // Mock the queue
- LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- //unset maxCapacity
- a.setMaxCapacity(1.0f);
-
- // Users
- final String user_0 = "user_0";
- final String user_1 = "user_1";
-
- // Submit applications
- final ApplicationAttemptId appAttemptId_0 =
- TestUtils.getMockApplicationAttemptId(0, 0);
- FiCaSchedulerApp app_0 =
- new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_0, user_0);
-
- final ApplicationAttemptId appAttemptId_1 =
- TestUtils.getMockApplicationAttemptId(1, 0);
- FiCaSchedulerApp app_1 =
- new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_1, user_0); // same user
-
- final ApplicationAttemptId appAttemptId_2 =
- TestUtils.getMockApplicationAttemptId(2, 0);
- FiCaSchedulerApp app_2 =
- new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_2, user_1);
-
- // Setup some nodes
- String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK,
- 0, 16*GB);
- String host_1 = "127.0.0.2";
- FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK,
- 0, 16*GB);
-
- Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
- app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
- app_1, app_2.getApplicationAttemptId(), app_2);
- Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
- node_0, node_1.getNodeID(), node_1);
-
- final int numNodes = 2;
- Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1);
- when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
- Priority priority = TestUtils.createMockPriority(1);
-
- app_0.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(1*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
- //Now, headroom is the same for all apps for a given user + queue combo
- //and a change to any app's headroom is reflected for all the user's apps
- //once those apps are active/have themselves calculated headroom for
- //allocation at least one time
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- app_1.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(2*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- //Complete container and verify that headroom is updated, for both apps
- //for the user
- RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
- a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- ContainerStatus.newInstance(rmContainer.getContainerId(),
- ContainerState.COMPLETE, "",
- ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
- RMContainerEventType.KILL, null, true);
-
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(2*GB, app_1.getHeadroom().getMemorySize());
- }
-
- @Test
public void testHeadroomWithMaxCap() throws Exception {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
@@ -1403,7 +1315,12 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
+
+ ParentQueue root = (ParentQueue) queues
+ .get(CapacitySchedulerConfiguration.ROOT);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
@@ -1454,6 +1371,8 @@ public class TestLeafQueue {
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
priority, recordFactory)));
@@ -1542,6 +1461,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1624,6 +1545,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1638,6 +1561,8 @@ public class TestLeafQueue {
// Now, allocations should goto app_3 since it's under user-limit
a.setMaxCapacity(1.0f);
a.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1743,6 +1668,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1880,6 +1807,8 @@ public class TestLeafQueue {
final int numNodes = 3;
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getMaximumResourceCapability()).thenReturn(
Resources.createResource(4*GB, 16));
@@ -2051,6 +1980,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2237,11 +2168,10 @@ public class TestLeafQueue {
CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// Manipulate queue 'b'
- LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
+ LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B));
// Check locality parameters.
assertEquals(2, a.getNodeLocalityDelay());
@@ -2277,6 +2207,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8 * GB), numNodes * 16);
when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes);
+ newRoot.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2412,6 +2344,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
@@ -2545,6 +2479,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2660,17 +2596,14 @@ public class TestLeafQueue {
assertEquals(2, e.getNumActiveApplications());
assertEquals(1, e.getNumPendingApplications());
- csConf.setDouble(CapacitySchedulerConfiguration
- .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
- CapacitySchedulerConfiguration
- .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
+ csConf.setDouble(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
+ * 2);
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
- CSQueue newRoot =
- CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
- CapacitySchedulerConfiguration.ROOT,
- newQueues, queues,
- TestUtils.spyHook);
- queues = newQueues;
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
root.reinitialize(newRoot, csContext.getClusterResource());
// after reinitialization
@@ -2697,7 +2630,6 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// after reinitialization
@@ -2745,7 +2677,7 @@ public class TestLeafQueue {
assertEquals(1, e.getNumPendingApplications());
Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32);
- e.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
// after updating cluster resource
@@ -2837,6 +2769,9 @@ public class TestLeafQueue {
numNodes * (8*GB), numNodes * 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
// resourceName: <priority, memory, #containers, relaxLocality>
// host_0_0: < 1, 1GB, 1, true >
@@ -3036,36 +2971,44 @@ public class TestLeafQueue {
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
throws Exception {
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- Resource clusterResource = Resources
- .createResource(100 * 16 * GB, 100 * 32);
+ final String newRootName = "root" + System.currentTimeMillis();
+ setupQueueConfiguration(csConf, newRootName);
+
+ Resource clusterResource = Resources.createResource(100 * 16 * GB,
+ 100 * 32);
CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
when(csContext.getRMContext()).thenReturn(rmContext);
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
- ParentQueue root = new ParentQueue(csContext,
- CapacitySchedulerConfiguration.ROOT, null, null);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
- LeafQueue a = new LeafQueue(csContext, A, root, null);
- assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(160 * GB, 1));
-
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
- LeafQueue newA = new LeafQueue(csContext, A, root, null);
- a.reinitialize(newA, clusterResource);
- assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(320 * GB, 1));
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.1f);
+
+ CSQueue root;
+ root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
- Resource newClusterResource = Resources.createResource(100 * 20 * GB,
- 100 * 32);
- a.updateClusterResource(newClusterResource,
- new ResourceLimits(newClusterResource));
- // 100 * 20 * 0.2 = 400
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(400 * GB, 1));
+ // Manipulate queue 'a'
+ LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B));
+ assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(159 * GB, 1));
+
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.2f);
+ clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
+ root.reinitialize(newRoot, clusterResource);
+
+ b = stubLeafQueue((LeafQueue) newQueues.get(B));
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(320 * GB, 1));
}
@Test
@@ -3142,6 +3085,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3308,6 +3253,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (16*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3435,6 +3382,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
// App0 has node local request for host_0/host_1, and app1 has node local
@@ -3533,6 +3482,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for app_0 and app_1 total 5GB.
Priority priority = TestUtils.createMockPriority(1);
@@ -3699,6 +3650,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for user_0: app_0 and app_1 total 3GB.
Priority priority = TestUtils.createMockPriority(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index a9196d1..25a9774 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -249,6 +249,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -494,6 +496,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
@@ -710,6 +714,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -790,6 +796,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue b3 = (LeafQueue)queues.get(B3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 32f022f..0855720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -267,6 +267,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -454,6 +456,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -600,6 +604,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -782,6 +788,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -896,6 +904,10 @@ public class TestReservations {
String host_1 = "host_1";
FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
8 * GB);
+
+ Resource clusterResource = Resources.createResource(2 * 8 * GB);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
@@ -1068,6 +1080,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1256,6 +1270,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1418,6 +1434,9 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-7386. Duplicate Strings in
various places in Yarn memory (misha@cloudera.com via rkanter)
Posted by su...@apache.org.
YARN-7386. Duplicate Strings in various places in Yarn memory (misha@cloudera.com via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2c150a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2c150a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2c150a7
Branch: refs/heads/YARN-5881
Commit: a2c150a7369cc629bbfaa2dfa3a8495b6f9c42e2
Parents: ba81366
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Nov 9 12:07:46 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Nov 9 12:12:52 2017 -0800
----------------------------------------------------------------------
.../impl/pb/ContainerLaunchContextPBImpl.java | 16 ++++++++++++----
.../yarn/api/records/impl/pb/ContainerPBImpl.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 7 ++++---
.../rmapp/attempt/RMAppAttemptImpl.java | 3 ++-
4 files changed, 19 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
index d722cc5..a9f2ee3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
@@ -392,7 +393,8 @@ extends ContainerLaunchContext {
this.environment = new HashMap<String, String>();
for (StringStringMapProto c : list) {
- this.environment.put(c.getKey(), c.getValue());
+ this.environment.put(StringInterner.weakIntern(c.getKey()),
+ StringInterner.weakIntern(c.getValue()));
}
}
@@ -402,7 +404,10 @@ extends ContainerLaunchContext {
return;
initEnv();
this.environment.clear();
- this.environment.putAll(env);
+ for (Map.Entry<String, String> e : env.entrySet()) {
+ this.environment.put(StringInterner.weakIntern(e.getKey()),
+ StringInterner.weakIntern(e.getValue()));
+ }
}
private void addEnvToProto() {
@@ -464,7 +469,7 @@ extends ContainerLaunchContext {
for (ApplicationACLMapProto aclProto : list) {
this.applicationACLS.put(ProtoUtils.convertFromProtoFormat(aclProto
- .getAccessType()), aclProto.getAcl());
+ .getAccessType()), StringInterner.weakIntern(aclProto.getAcl()));
}
}
@@ -513,7 +518,10 @@ extends ContainerLaunchContext {
return;
initApplicationACLs();
this.applicationACLS.clear();
- this.applicationACLS.putAll(appACLs);
+ for (Map.Entry<ApplicationAccessType, String> e : appACLs.entrySet()) {
+ this.applicationACLS.put(e.getKey(),
+ StringInterner.weakIntern(e.getValue()));
+ }
}
public ContainerRetryContext getContainerRetryContext() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index b6e22d1..be84938 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -181,7 +181,7 @@ public class ContainerPBImpl extends Container {
builder.clearNodeHttpAddress();
return;
}
- builder.setNodeHttpAddress(nodeHttpAddress);
+ builder.setNodeHttpAddress(nodeHttpAddress.intern());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index cfb8a74..ae5f6b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -425,12 +426,12 @@ public class RMAppImpl implements RMApp, Recoverable {
this.systemClock = SystemClock.getInstance();
this.applicationId = applicationId;
- this.name = name;
+ this.name = StringInterner.weakIntern(name);
this.rmContext = rmContext;
this.dispatcher = rmContext.getDispatcher();
this.handler = dispatcher.getEventHandler();
this.conf = config;
- this.user = user;
+ this.user = StringInterner.weakIntern(user);
this.queue = queue;
this.submissionContext = submissionContext;
this.scheduler = scheduler;
@@ -441,7 +442,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
- this.applicationType = applicationType;
+ this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
if (submissionContext.getPriority() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c150a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index d3fbcb5..2d5c6a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -1614,7 +1615,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
RMAppAttemptRegistrationEvent registrationEvent
= (RMAppAttemptRegistrationEvent) event;
- appAttempt.host = registrationEvent.getHost();
+ appAttempt.host = StringInterner.weakIntern(registrationEvent.getHost());
appAttempt.rpcPort = registrationEvent.getRpcport();
appAttempt.originalTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-7394. Merge code paths for
Reservation/Plan queues and Auto Created queues. (Suma Shivaprasad via
wangda)
Posted by su...@apache.org.
YARN-7394. Merge code paths for Reservation/Plan queues and Auto Created queues. (Suma Shivaprasad via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13fa2d4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13fa2d4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13fa2d4e
Branch: refs/heads/YARN-5881
Commit: 13fa2d4e3e55a849dcd7e472750f3e0422cc2ac9
Parents: 8f214dc
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 6 21:38:24 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Nov 6 21:38:24 2017 -0800
----------------------------------------------------------------------
.../CapacitySchedulerPlanFollower.java | 10 +-
.../capacity/AbstractManagedParentQueue.java | 232 +++++++++++++++++++
.../capacity/AutoCreatedLeafQueue.java | 129 +++++++++++
.../scheduler/capacity/CapacityScheduler.java | 55 +++--
.../capacity/CapacitySchedulerQueueManager.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 4 +
.../scheduler/capacity/ParentQueue.java | 13 ++
.../scheduler/capacity/PlanQueue.java | 191 ++-------------
.../scheduler/capacity/ReservationQueue.java | 122 ----------
.../capacity/TestAutoCreatedLeafQueue.java | 113 +++++++++
.../TestCapacitySchedulerDynamicBehavior.java | 32 +--
.../capacity/TestReservationQueue.java | 110 ---------
12 files changed, 569 insertions(+), 446 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
index 551f075..2e16689 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
@@ -28,10 +28,10 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ReservationQueue;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
@@ -92,8 +92,8 @@ public class CapacitySchedulerPlanFollower extends AbstractSchedulerPlanFollower
String planQueueName, Queue queue, String currResId) {
PlanQueue planQueue = (PlanQueue)queue;
try {
- ReservationQueue resQueue =
- new ReservationQueue(cs, currResId, planQueue);
+ AutoCreatedLeafQueue resQueue =
+ new AutoCreatedLeafQueue(cs, currResId, planQueue);
cs.addQueue(resQueue);
} catch (SchedulerDynamicEditException e) {
LOG.warn(
@@ -112,8 +112,8 @@ public class CapacitySchedulerPlanFollower extends AbstractSchedulerPlanFollower
PlanQueue planQueue = (PlanQueue)queue;
if (cs.getQueue(defReservationId) == null) {
try {
- ReservationQueue defQueue =
- new ReservationQueue(cs, defReservationId, planQueue);
+ AutoCreatedLeafQueue defQueue =
+ new AutoCreatedLeafQueue(cs, defReservationId, planQueue);
cs.addQueue(defQueue);
} catch (SchedulerDynamicEditException e) {
LOG.warn(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
new file mode 100644
index 0000000..b3d1b47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * A container class for automatically created child leaf queues.
+ * From the user perspective this is equivalent to a LeafQueue,
+ * but functionality wise is a sub-class of ParentQueue
+ */
+public abstract class AbstractManagedParentQueue extends ParentQueue {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AbstractManagedParentQueue.class);
+
+ private int maxAppsForAutoCreatedQueues;
+ private int maxAppsPerUserForAutoCreatedQueues;
+ private int userLimit;
+ private float userLimitFactor;
+
+ public AbstractManagedParentQueue(CapacitySchedulerContext cs,
+ String queueName, CSQueue parent, CSQueue old) throws IOException {
+ super(cs, queueName, parent, old);
+
+ super.setupQueueConfigs(csContext.getClusterResource());
+ initializeLeafQueueConfigs();
+
+ StringBuffer queueInfo = new StringBuffer();
+ queueInfo.append("Created Managed Parent Queue: ").append(queueName)
+ .append("\nof type : [" + getClass())
+ .append("]\nwith capacity: [")
+ .append(super.getCapacity()).append("]\nwith max capacity: [")
+ .append(super.getMaximumCapacity()).append("\nwith max apps: [")
+ .append(getMaxApplicationsForAutoCreatedQueues())
+ .append("]\nwith max apps per user: [")
+ .append(getMaxApplicationsPerUserForAutoCreatedQueues())
+ .append("]\nwith user limit: [").append(getUserLimit())
+ .append("]\nwith user limit factor: [")
+ .append(getUserLimitFactor()).append("].");
+ LOG.info(queueInfo.toString());
+ }
+
+ @Override
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
+ try {
+ writeLock.lock();
+
+ // Set new configs
+ setupQueueConfigs(clusterResource);
+
+ initializeLeafQueueConfigs();
+
+ // run reinitialize on each existing queue, to trigger absolute cap
+ // recomputations
+ for (CSQueue res : this.getChildQueues()) {
+ res.reinitialize(res, clusterResource);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Initialize leaf queue configs from template configurations specified on
+ * parent queue.
+ */
+ protected void initializeLeafQueueConfigs() {
+
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+
+ final String queuePath = super.getQueuePath();
+ int maxApps = conf.getMaximumApplicationsPerQueue(queuePath);
+ if (maxApps < 0) {
+ maxApps = (int) (
+ CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
+ * getAbsoluteCapacity());
+ }
+ userLimit = conf.getUserLimit(queuePath);
+ userLimitFactor = conf.getUserLimitFactor(queuePath);
+ maxAppsForAutoCreatedQueues = maxApps;
+ maxAppsPerUserForAutoCreatedQueues =
+ (int) (maxApps * (userLimit / 100.0f) * userLimitFactor);
+
+ }
+
+ /**
+ * Number of maximum applications for each of the auto created leaf queues.
+ *
+ * @return maxAppsForAutoCreatedQueues
+ */
+ public int getMaxApplicationsForAutoCreatedQueues() {
+ return maxAppsForAutoCreatedQueues;
+ }
+
+ /**
+ * Number of maximum applications per user for each of the auto created
+ * leaf queues.
+ *
+ * @return maxAppsPerUserForAutoCreatedQueues
+ */
+ public int getMaxApplicationsPerUserForAutoCreatedQueues() {
+ return maxAppsPerUserForAutoCreatedQueues;
+ }
+
+ /**
+ * User limit value for each of the auto created leaf queues.
+ *
+ * @return userLimit
+ */
+ public int getUserLimitForAutoCreatedQueues() {
+ return userLimit;
+ }
+
+ /**
+ * User limit factor value for each of the auto created leaf queues.
+ *
+ * @return userLimitFactor
+ */
+ public float getUserLimitFactor() {
+ return userLimitFactor;
+ }
+
+ public int getMaxAppsForAutoCreatedQueues() {
+ return maxAppsForAutoCreatedQueues;
+ }
+
+ public int getMaxAppsPerUserForAutoCreatedQueues() {
+ return maxAppsPerUserForAutoCreatedQueues;
+ }
+
+ public int getUserLimit() {
+ return userLimit;
+ }
+
+ /**
+ * Add the specified child queue.
+ * @param childQueue reference to the child queue to be added
+ * @throws SchedulerDynamicEditException
+ */
+ public void addChildQueue(CSQueue childQueue)
+ throws SchedulerDynamicEditException {
+ try {
+ writeLock.lock();
+ if (childQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException(
+ "Queue " + childQueue + " being added has non zero capacity.");
+ }
+ boolean added = this.childQueues.add(childQueue);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateChildQueues (action: add queue): " + added + " "
+ + getChildQueuesToPrint());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Remove the specified child queue.
+ * @param childQueue reference to the child queue to be removed
+ * @throws SchedulerDynamicEditException
+ */
+ public void removeChildQueue(CSQueue childQueue)
+ throws SchedulerDynamicEditException {
+ try {
+ writeLock.lock();
+ if (childQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException(
+ "Queue " + childQueue + " being removed has non zero capacity.");
+ }
+ Iterator<CSQueue> qiter = childQueues.iterator();
+ while (qiter.hasNext()) {
+ CSQueue cs = qiter.next();
+ if (cs.equals(childQueue)) {
+ qiter.remove();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removed child queue: {}" + cs.getQueueName());
+ }
+ }
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Remove the specified child queue.
+ * @param childQueueName name of the child queue to be removed
+ * @throws SchedulerDynamicEditException
+ */
+ public CSQueue removeChildQueue(String childQueueName)
+ throws SchedulerDynamicEditException {
+ CSQueue childQueue;
+ try {
+ writeLock.lock();
+ childQueue = this.csContext.getCapacitySchedulerQueueManager().getQueue(
+ childQueueName);
+ if (childQueue != null) {
+ removeChildQueue(childQueue);
+ } else {
+ throw new SchedulerDynamicEditException("Cannot find queue to delete "
+ + ": " + childQueueName);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ return childQueue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
new file mode 100644
index 0000000..4eb7cdd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Leaf queues which are auto created by an underkying implementation of
+ * AbstractManagedParentQueue. Eg: PlanQueue for reservations or
+ * ManagedParentQueue for auto created dynamic queues
+ */
+public class AutoCreatedLeafQueue extends LeafQueue {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(AutoCreatedLeafQueue.class);
+
+ private AbstractManagedParentQueue parent;
+
+ public AutoCreatedLeafQueue(CapacitySchedulerContext cs, String queueName,
+ AbstractManagedParentQueue parent) throws IOException {
+ super(cs, queueName, parent, null);
+
+ updateApplicationAndUserLimits(parent.getUserLimitForAutoCreatedQueues(),
+ parent.getUserLimitFactor(),
+ parent.getMaxApplicationsForAutoCreatedQueues(),
+ parent.getMaxApplicationsPerUserForAutoCreatedQueues());
+
+ this.parent = parent;
+ }
+
+ @Override
+ public void reinitialize(CSQueue newlyParsedQueue,
+ Resource clusterResource) throws IOException {
+ try {
+ writeLock.lock();
+
+ validate(newlyParsedQueue);
+
+ super.reinitialize(newlyParsedQueue, clusterResource);
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ this, labelManager, null);
+
+ updateApplicationAndUserLimits(parent.getUserLimitForAutoCreatedQueues(),
+ parent.getUserLimitFactor(),
+ parent.getMaxApplicationsForAutoCreatedQueues(),
+ parent.getMaxApplicationsPerUserForAutoCreatedQueues());
+
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * This methods to change capacity for a queue and adjusts its
+ * absoluteCapacity.
+ *
+ * @param entitlement the new entitlement for the queue (capacity,
+ * maxCapacity)
+ * @throws SchedulerDynamicEditException
+ */
+ public void setEntitlement(QueueEntitlement entitlement)
+ throws SchedulerDynamicEditException {
+ try {
+ writeLock.lock();
+ float capacity = entitlement.getCapacity();
+ if (capacity < 0 || capacity > 1.0f) {
+ throw new SchedulerDynamicEditException(
+ "Capacity demand is not in the [0,1] range: " + capacity);
+ }
+ setCapacity(capacity);
+ setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
+ setMaxCapacity(entitlement.getMaxCapacity());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully changed to " + capacity + " for queue " + this
+ .getQueueName());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void validate(final CSQueue newlyParsedQueue) throws IOException {
+ if (!(newlyParsedQueue instanceof AutoCreatedLeafQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Error trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
+
+ }
+
+ @Override
+ protected void setupConfigurableCapacities() {
+ CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
+ queueCapacities, parent == null ? null : parent.getQueueCapacities());
+ }
+
+ private void updateApplicationAndUserLimits(int userLimit,
+ float userLimitFactor,
+ int maxAppsForAutoCreatedQueues,
+ int maxAppsPerUserForAutoCreatedQueues) {
+ setUserLimit(userLimit);
+ setUserLimitFactor(userLimitFactor);
+ setMaxApplications(maxAppsForAutoCreatedQueues);
+ setMaxApplicationsPerUser(maxAppsPerUserForAutoCreatedQueues);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index ca289b1..db69042 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1921,12 +1921,12 @@ public class CapacityScheduler extends
writeLock.lock();
LOG.info("Removing queue: " + queueName);
CSQueue q = this.getQueue(queueName);
- if (!(q instanceof ReservationQueue)) {
+ if (!(q instanceof AutoCreatedLeafQueue)) {
throw new SchedulerDynamicEditException(
"The queue that we are asked " + "to remove (" + queueName
- + ") is not a ReservationQueue");
+ + ") is not a AutoCreatedLeafQueue");
}
- ReservationQueue disposableLeafQueue = (ReservationQueue) q;
+ AutoCreatedLeafQueue disposableLeafQueue = (AutoCreatedLeafQueue) q;
// at this point we should have no more apps
if (disposableLeafQueue.getNumApplications() > 0) {
throw new SchedulerDynamicEditException(
@@ -1936,9 +1936,11 @@ public class CapacityScheduler extends
+ " pending apps");
}
- ((PlanQueue) disposableLeafQueue.getParent()).removeChildQueue(q);
+ ((AbstractManagedParentQueue) disposableLeafQueue.getParent())
+ .removeChildQueue(q);
this.queueManager.removeQueue(queueName);
- LOG.info("Removal of ReservationQueue " + queueName + " has succeeded");
+ LOG.info("Removal of AutoCreatedLeafQueue "
+ + queueName + " has succeeded");
} finally {
writeLock.unlock();
}
@@ -1949,25 +1951,28 @@ public class CapacityScheduler extends
throws SchedulerDynamicEditException {
try {
writeLock.lock();
- if (!(queue instanceof ReservationQueue)) {
+ if (!(queue instanceof AutoCreatedLeafQueue)) {
throw new SchedulerDynamicEditException(
- "Queue " + queue.getQueueName() + " is not a ReservationQueue");
+ "Queue " + queue.getQueueName() + " is not a AutoCreatedLeafQueue");
}
- ReservationQueue newQueue = (ReservationQueue) queue;
+ AutoCreatedLeafQueue newQueue = (AutoCreatedLeafQueue) queue;
- if (newQueue.getParent() == null || !(newQueue
- .getParent() instanceof PlanQueue)) {
+ if (newQueue.getParent() == null
+ || !(AbstractManagedParentQueue.class.
+ isAssignableFrom(newQueue.getParent().getClass()))) {
throw new SchedulerDynamicEditException(
"ParentQueue for " + newQueue.getQueueName()
- + " is not properly set (should be set and be a PlanQueue)");
+ + " is not properly set"
+ + " (should be set and be a PlanQueue or ManagedParentQueue)");
}
- PlanQueue parentPlan = (PlanQueue) newQueue.getParent();
+ AbstractManagedParentQueue parentPlan =
+ (AbstractManagedParentQueue) newQueue.getParent();
String queuename = newQueue.getQueueName();
parentPlan.addChildQueue(newQueue);
this.queueManager.addQueue(queuename, newQueue);
- LOG.info("Creation of ReservationQueue " + newQueue + " succeeded");
+ LOG.info("Creation of AutoCreatedLeafQueue " + newQueue + " succeeded");
} finally {
writeLock.unlock();
}
@@ -1981,21 +1986,22 @@ public class CapacityScheduler extends
LeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue);
ParentQueue parent = (ParentQueue) queue.getParent();
- if (!(queue instanceof ReservationQueue)) {
+ if (!(queue instanceof AutoCreatedLeafQueue)) {
throw new SchedulerDynamicEditException(
"Entitlement can not be" + " modified dynamically since queue "
- + inQueue + " is not a ReservationQueue");
+ + inQueue + " is not a AutoCreatedLeafQueue");
}
- if (!(parent instanceof PlanQueue)) {
+ if (parent == null
+ || !(AbstractManagedParentQueue.class.isAssignableFrom(parent.getClass()))) {
throw new SchedulerDynamicEditException(
- "The parent of ReservationQueue " + inQueue
- + " must be an PlanQueue");
+ "The parent of AutoCreatedLeafQueue " + inQueue
+ + " must be a PlanQueue/ManagedParentQueue");
}
- ReservationQueue newQueue = (ReservationQueue) queue;
+ AutoCreatedLeafQueue newQueue = (AutoCreatedLeafQueue) queue;
- float sumChilds = ((PlanQueue) parent).sumOfChildCapacities();
+ float sumChilds = parent.sumOfChildCapacities();
float newChildCap =
sumChilds - queue.getCapacity() + entitlement.getCapacity();
@@ -2010,12 +2016,13 @@ public class CapacityScheduler extends
newQueue.setEntitlement(entitlement);
} else{
throw new SchedulerDynamicEditException(
- "Sum of child queues would exceed 100% for PlanQueue: " + parent
- .getQueueName());
+ "Sum of child queues should exceed 100% for auto creating parent "
+ + "queue : " + parent.getQueueName());
}
LOG.info(
- "Set entitlement for ReservationQueue " + inQueue + " to " + queue
- .getCapacity() + " request was (" + entitlement.getCapacity()
+ "Set entitlement for AutoCreatedLeafQueue " + inQueue
+ + " to " + queue.getCapacity() +
+ " request was (" + entitlement.getCapacity()
+ ")");
} finally {
writeLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 48c289f..7be2529 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -238,7 +238,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
List<CSQueue> childQueues = new ArrayList<>();
- ReservationQueue resQueue = new ReservationQueue(csContext,
+ AutoCreatedLeafQueue resQueue = new AutoCreatedLeafQueue(csContext,
defReservationId, (PlanQueue) queue);
try {
resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
@@ -303,7 +303,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
Map<String, CSQueue> newQueues) throws IOException {
// check that all static queues are included in the newQueues list
for (Map.Entry<String, CSQueue> e : queues.entrySet()) {
- if (!(e.getValue() instanceof ReservationQueue)) {
+ if (!(e.getValue() instanceof AutoCreatedLeafQueue)) {
String queueName = e.getKey();
CSQueue oldQueue = e.getValue();
CSQueue newQueue = newQueues.get(queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f24e30a..f2f1baf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1997,6 +1997,10 @@ public class LeafQueue extends AbstractCSQueue {
queueCapacities.setAbsoluteCapacity(absoluteCapacity);
}
+ public void setMaxApplicationsPerUser(int maxApplicationsPerUser) {
+ this.maxApplicationsPerUser = maxApplicationsPerUser;
+ }
+
public void setMaxApplications(int maxApplications) {
this.maxApplications = maxApplications;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 6800b74..2c288f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1080,4 +1080,17 @@ public class ParentQueue extends AbstractCSQueue {
public QueueOrderingPolicy getQueueOrderingPolicy() {
return queueOrderingPolicy;
}
+
+ protected float sumOfChildCapacities() {
+ try {
+ writeLock.lock();
+ float ret = 0;
+ for (CSQueue l : childQueues) {
+ ret += l.getCapacity();
+ }
+ return ret;
+ } finally {
+ writeLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 882262f..4ab2e9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -19,11 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.util.Iterator;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,189 +31,48 @@ import org.slf4j.LoggerFactory;
* reservations, but functionality wise is a sub-class of ParentQueue
*
*/
-public class PlanQueue extends ParentQueue {
+public class PlanQueue extends AbstractManagedParentQueue {
private static final Logger LOG = LoggerFactory.getLogger(PlanQueue.class);
- private int maxAppsForReservation;
- private int maxAppsPerUserForReservation;
- private int userLimit;
- private float userLimitFactor;
- protected CapacitySchedulerContext schedulerContext;
private boolean showReservationsAsQueues;
public PlanQueue(CapacitySchedulerContext cs, String queueName,
CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
-
- this.schedulerContext = cs;
- // Set the reservation queue attributes for the Plan
- CapacitySchedulerConfiguration conf = cs.getConfiguration();
- String queuePath = super.getQueuePath();
- int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath);
- showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath);
- if (maxAppsForReservation < 0) {
- maxAppsForReservation =
- (int) (CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super
- .getAbsoluteCapacity());
- }
- int userLimit = conf.getUserLimit(queuePath);
- float userLimitFactor = conf.getUserLimitFactor(queuePath);
- int maxAppsPerUserForReservation =
- (int) (maxAppsForReservation * (userLimit / 100.0f) * userLimitFactor);
- updateQuotas(userLimit, userLimitFactor, maxAppsForReservation,
- maxAppsPerUserForReservation);
-
- StringBuffer queueInfo = new StringBuffer();
- queueInfo.append("Created Plan Queue: ").append(queueName)
- .append("\nwith capacity: [").append(super.getCapacity())
- .append("]\nwith max capacity: [").append(super.getMaximumCapacity())
- .append("\nwith max reservation apps: [").append(maxAppsForReservation)
- .append("]\nwith max reservation apps per user: [")
- .append(maxAppsPerUserForReservation).append("]\nwith user limit: [")
- .append(userLimit).append("]\nwith user limit factor: [")
- .append(userLimitFactor).append("].");
- LOG.info(queueInfo.toString());
}
@Override
- public void reinitialize(CSQueue newlyParsedQueue,
- Resource clusterResource) throws IOException {
- try {
- writeLock.lock();
- // Sanity check
- if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
- .getQueuePath().equals(getQueuePath())) {
- throw new IOException(
- "Trying to reinitialize " + getQueuePath() + " from "
- + newlyParsedQueue.getQueuePath());
- }
-
- PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
-
- if (newlyParsedParentQueue.getChildQueues().size() != 1) {
- throw new IOException(
- "Reservable Queue should not have sub-queues in the"
- + "configuration expect the default reservation queue");
- }
-
- // Set new configs
- setupQueueConfigs(clusterResource);
-
- updateQuotas(newlyParsedParentQueue.userLimit,
- newlyParsedParentQueue.userLimitFactor,
- newlyParsedParentQueue.maxAppsForReservation,
- newlyParsedParentQueue.maxAppsPerUserForReservation);
-
- // run reinitialize on each existing queue, to trigger absolute cap
- // recomputations
- for (CSQueue res : this.getChildQueues()) {
- res.reinitialize(res, clusterResource);
- }
- showReservationsAsQueues =
- newlyParsedParentQueue.showReservationsAsQueues;
- } finally {
- writeLock.unlock();
- }
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
+ validate(newlyParsedQueue);
+ super.reinitialize(newlyParsedQueue, clusterResource);
}
- void addChildQueue(CSQueue newQueue)
- throws SchedulerDynamicEditException {
- try {
- writeLock.lock();
- if (newQueue.getCapacity() > 0) {
- throw new SchedulerDynamicEditException(
- "Queue " + newQueue + " being added has non zero capacity.");
- }
- boolean added = this.childQueues.add(newQueue);
- if (LOG.isDebugEnabled()) {
- LOG.debug("updateChildQueues (action: add queue): " + added + " "
- + getChildQueuesToPrint());
- }
- } finally {
- writeLock.unlock();
- }
+ @Override
+ protected void initializeLeafQueueConfigs() {
+ String queuePath = super.getQueuePath();
+ showReservationsAsQueues = csContext.getConfiguration()
+ .getShowReservationAsQueues(queuePath);
+ super.initializeLeafQueueConfigs();
}
- void removeChildQueue(CSQueue remQueue)
- throws SchedulerDynamicEditException {
- try {
- writeLock.lock();
- if (remQueue.getCapacity() > 0) {
- throw new SchedulerDynamicEditException(
- "Queue " + remQueue + " being removed has non zero capacity.");
- }
- Iterator<CSQueue> qiter = childQueues.iterator();
- while (qiter.hasNext()) {
- CSQueue cs = qiter.next();
- if (cs.equals(remQueue)) {
- qiter.remove();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removed child queue: {}", cs.getQueueName());
- }
- }
- }
- } finally {
- writeLock.unlock();
+ private void validate(final CSQueue newlyParsedQueue) throws IOException {
+ // Sanity check
+ if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
}
- }
- protected float sumOfChildCapacities() {
- try {
- writeLock.lock();
- float ret = 0;
- for (CSQueue l : childQueues) {
- ret += l.getCapacity();
- }
- return ret;
- } finally {
- writeLock.unlock();
- }
- }
+ PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
- private void updateQuotas(int userLimit, float userLimitFactor,
- int maxAppsForReservation, int maxAppsPerUserForReservation) {
- this.userLimit = userLimit;
- this.userLimitFactor = userLimitFactor;
- this.maxAppsForReservation = maxAppsForReservation;
- this.maxAppsPerUserForReservation = maxAppsPerUserForReservation;
- }
-
- /**
- * Number of maximum applications for each of the reservations in this Plan.
- *
- * @return maxAppsForreservation
- */
- public int getMaxApplicationsForReservations() {
- return maxAppsForReservation;
- }
-
- /**
- * Number of maximum applications per user for each of the reservations in
- * this Plan.
- *
- * @return maxAppsPerUserForreservation
- */
- public int getMaxApplicationsPerUserForReservation() {
- return maxAppsPerUserForReservation;
- }
-
- /**
- * User limit value for each of the reservations in this Plan.
- *
- * @return userLimit
- */
- public int getUserLimitForReservation() {
- return userLimit;
- }
-
- /**
- * User limit factor value for each of the reservations in this Plan.
- *
- * @return userLimitFactor
- */
- public float getUserLimitFactor() {
- return userLimitFactor;
+ if (newlyParsedParentQueue.getChildQueues().size() != 1) {
+ throw new IOException(
+ "Reservable Queue should not have sub-queues in the"
+ + "configuration expect the default reservation queue");
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
deleted file mode 100644
index 3d1b731..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This represents a dynamic {@link LeafQueue} managed by the
- * {@link ReservationSystem}
- *
- */
-public class ReservationQueue extends LeafQueue {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(ReservationQueue.class);
-
- private PlanQueue parent;
-
- public ReservationQueue(CapacitySchedulerContext cs, String queueName,
- PlanQueue parent) throws IOException {
- super(cs, queueName, parent, null);
- // the following parameters are common to all reservation in the plan
- updateQuotas(parent.getUserLimitForReservation(),
- parent.getUserLimitFactor(),
- parent.getMaxApplicationsForReservations(),
- parent.getMaxApplicationsPerUserForReservation());
- this.parent = parent;
- }
-
- @Override
- public void reinitialize(CSQueue newlyParsedQueue,
- Resource clusterResource) throws IOException {
- try {
- writeLock.lock();
- // Sanity check
- if (!(newlyParsedQueue instanceof ReservationQueue) || !newlyParsedQueue
- .getQueuePath().equals(getQueuePath())) {
- throw new IOException(
- "Trying to reinitialize " + getQueuePath() + " from "
- + newlyParsedQueue.getQueuePath());
- }
- super.reinitialize(newlyParsedQueue, clusterResource);
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- this, labelManager, null);
-
- updateQuotas(parent.getUserLimitForReservation(),
- parent.getUserLimitFactor(),
- parent.getMaxApplicationsForReservations(),
- parent.getMaxApplicationsPerUserForReservation());
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * This methods to change capacity for a queue and adjusts its
- * absoluteCapacity
- *
- * @param entitlement the new entitlement for the queue (capacity,
- * maxCapacity, etc..)
- * @throws SchedulerDynamicEditException
- */
- public void setEntitlement(QueueEntitlement entitlement)
- throws SchedulerDynamicEditException {
- try {
- writeLock.lock();
- float capacity = entitlement.getCapacity();
- if (capacity < 0 || capacity > 1.0f) {
- throw new SchedulerDynamicEditException(
- "Capacity demand is not in the [0,1] range: " + capacity);
- }
- setCapacity(capacity);
- setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
- // note: we currently set maxCapacity to capacity
- // this might be revised later
- setMaxCapacity(entitlement.getMaxCapacity());
- if (LOG.isDebugEnabled()) {
- LOG.debug("successfully changed to " + capacity + " for queue " + this
- .getQueueName());
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- private void updateQuotas(int userLimit, float userLimitFactor,
- int maxAppsForReservation, int maxAppsPerUserForReservation) {
- setUserLimit(userLimit);
- setUserLimitFactor(userLimitFactor);
- setMaxApplications(maxAppsForReservation);
- maxApplicationsPerUser = maxAppsPerUserForReservation;
- }
-
- @Override
- protected void setupConfigurableCapacities() {
- CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
- queueCapacities, parent == null ? null : parent.getQueueCapacities());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
new file mode 100644
index 0000000..b403e72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for dynamic auto created leaf queues.
+ * @see AutoCreatedLeafQueue
+ */
+public class TestAutoCreatedLeafQueue {
+
+ private CapacitySchedulerConfiguration csConf;
+ private CapacitySchedulerContext csContext;
+ final static int DEF_MAX_APPS = 10000;
+ final static int GB = 1024;
+ private final ResourceCalculator resourceCalculator =
+ new DefaultResourceCalculator();
+ private AutoCreatedLeafQueue autoCreatedLeafQueue;
+
+ @Before
+ public void setup() throws IOException {
+ // setup a context / conf
+ csConf = new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration();
+ csContext = mock(CapacitySchedulerContext.class);
+ when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getConf()).thenReturn(conf);
+ when(csContext.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(GB, 1));
+ when(csContext.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(16 * GB, 32));
+ when(csContext.getClusterResource()).thenReturn(
+ Resources.createResource(100 * 16 * GB, 100 * 32));
+ when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
+ RMContext mockRMContext = TestUtils.getMockRMContext();
+ when(csContext.getRMContext()).thenReturn(mockRMContext);
+
+ // create a queue
+ PlanQueue pq = new PlanQueue(csContext, "root", null, null);
+ autoCreatedLeafQueue = new AutoCreatedLeafQueue(csContext, "a", pq);
+ }
+
+ private void validateAutoCreatedLeafQueue(double capacity) {
+ assertTrue(" actual capacity: " + autoCreatedLeafQueue.getCapacity(),
+ autoCreatedLeafQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
+ assertEquals(autoCreatedLeafQueue.maxApplications, DEF_MAX_APPS);
+ assertEquals(autoCreatedLeafQueue.maxApplicationsPerUser, DEF_MAX_APPS);
+ }
+
+ @Test
+ public void testAddSubtractCapacity() throws Exception {
+
+ // verify that setting, adding, subtracting capacity works
+ autoCreatedLeafQueue.setCapacity(1.0F);
+ validateAutoCreatedLeafQueue(1);
+ autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
+ validateAutoCreatedLeafQueue(0.9);
+ autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1f, 1f));
+ validateAutoCreatedLeafQueue(1);
+ autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0f, 1f));
+ validateAutoCreatedLeafQueue(0);
+
+ try {
+ autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
+ fail();
+ } catch (SchedulerDynamicEditException iae) {
+ // expected
+ validateAutoCreatedLeafQueue(1);
+ }
+
+ try {
+ autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(-0.1f, 1f));
+ fail();
+ } catch (SchedulerDynamicEditException iae) {
+ // expected
+ validateAutoCreatedLeafQueue(1);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index 9aba30c..9425d5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -77,21 +77,21 @@ public class TestCapacitySchedulerDynamicBehavior {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
//set default queue capacity to zero
- ((ReservationQueue) cs
+ ((AutoCreatedLeafQueue) cs
.getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
.setEntitlement(
new QueueEntitlement(0f, 1f));
// Test add one reservation dynamically and manually modify capacity
- ReservationQueue a1 =
- new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a1 =
+ new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a1);
a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
// Test add another reservation queue and use setEntitlement to modify
// capacity
- ReservationQueue a2 =
- new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a2 =
+ new AutoCreatedLeafQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a2);
cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f));
@@ -113,8 +113,8 @@ public class TestCapacitySchedulerDynamicBehavior {
try {
// Test invalid addition (adding non-zero size queue)
- ReservationQueue a1 =
- new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a1 =
+ new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
cs.addQueue(a1);
fail();
@@ -123,11 +123,11 @@ public class TestCapacitySchedulerDynamicBehavior {
}
// Test add one reservation dynamically and manually modify capacity
- ReservationQueue a1 =
- new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a1 =
+ new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a1);
//set default queue capacity to zero
- ((ReservationQueue) cs
+ ((AutoCreatedLeafQueue) cs
.getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
.setEntitlement(
new QueueEntitlement(0f, 1f));
@@ -135,8 +135,8 @@ public class TestCapacitySchedulerDynamicBehavior {
// Test add another reservation queue and use setEntitlement to modify
// capacity
- ReservationQueue a2 =
- new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a2 =
+ new AutoCreatedLeafQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a2);
@@ -162,8 +162,8 @@ public class TestCapacitySchedulerDynamicBehavior {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
// Test add one reservation dynamically and manually modify capacity
- ReservationQueue a1 =
- new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+ AutoCreatedLeafQueue a1 =
+ new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a1);
a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
@@ -230,8 +230,8 @@ public class TestCapacitySchedulerDynamicBehavior {
// create the default reservation queue
String defQName = "a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
- ReservationQueue defQ =
- new ReservationQueue(scheduler, defQName,
+ AutoCreatedLeafQueue defQ =
+ new AutoCreatedLeafQueue(scheduler, defQName,
(PlanQueue) scheduler.getQueue("a"));
scheduler.addQueue(defQ);
defQ.setEntitlement(new QueueEntitlement(1f, 1f));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fa2d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
deleted file mode 100644
index e23e93c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestReservationQueue {
-
- CapacitySchedulerConfiguration csConf;
- CapacitySchedulerContext csContext;
- final static int DEF_MAX_APPS = 10000;
- final static int GB = 1024;
- private final ResourceCalculator resourceCalculator =
- new DefaultResourceCalculator();
- ReservationQueue reservationQueue;
-
- @Before
- public void setup() throws IOException {
- // setup a context / conf
- csConf = new CapacitySchedulerConfiguration();
- YarnConfiguration conf = new YarnConfiguration();
- csContext = mock(CapacitySchedulerContext.class);
- when(csContext.getConfiguration()).thenReturn(csConf);
- when(csContext.getConf()).thenReturn(conf);
- when(csContext.getMinimumResourceCapability()).thenReturn(
- Resources.createResource(GB, 1));
- when(csContext.getMaximumResourceCapability()).thenReturn(
- Resources.createResource(16 * GB, 32));
- when(csContext.getClusterResource()).thenReturn(
- Resources.createResource(100 * 16 * GB, 100 * 32));
- when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
-
- RMContext mockRMContext = TestUtils.getMockRMContext();
- when(csContext.getRMContext()).thenReturn(mockRMContext);
-
- // create a queue
- PlanQueue pq = new PlanQueue(csContext, "root", null, null);
- reservationQueue = new ReservationQueue(csContext, "a", pq);
- }
-
- private void validateReservationQueue(double capacity) {
- assertTrue(" actual capacity: " + reservationQueue.getCapacity(),
- reservationQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
- assertEquals(reservationQueue.maxApplications, DEF_MAX_APPS);
- assertEquals(reservationQueue.maxApplicationsPerUser, DEF_MAX_APPS);
- }
-
- @Test
- public void testAddSubtractCapacity() throws Exception {
-
- // verify that setting, adding, subtracting capacity works
- reservationQueue.setCapacity(1.0F);
- validateReservationQueue(1);
- reservationQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
- validateReservationQueue(0.9);
- reservationQueue.setEntitlement(new QueueEntitlement(1f, 1f));
- validateReservationQueue(1);
- reservationQueue.setEntitlement(new QueueEntitlement(0f, 1f));
- validateReservationQueue(0);
-
- try {
- reservationQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
- fail();
- } catch (SchedulerDynamicEditException iae) {
- // expected
- validateReservationQueue(1);
- }
-
- try {
- reservationQueue.setEntitlement(new QueueEntitlement(-0.1f, 1f));
- fail();
- } catch (SchedulerDynamicEditException iae) {
- // expected
- validateReservationQueue(1);
- }
-
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-7465. start-yarn.sh fails to
start ResourceManagers unless running as root.
Posted by su...@apache.org.
YARN-7465. start-yarn.sh fails to start ResourceManagers unless running as root.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1883a002
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1883a002
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1883a002
Branch: refs/heads/YARN-5881
Commit: 1883a0024949f7946264a4d7649b03fd1881567a
Parents: ac4d2b1
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Nov 8 15:45:35 2017 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Nov 9 14:47:43 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1883a002/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
index c009b43..b2244ec 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
@@ -65,7 +65,7 @@ else
RMHOSTS="${RMHOSTS} ${rmhost}"
done
echo "Starting resourcemanagers on [${RMHOSTS}]"
- hadoop_uservar_su yarn "${HADOOP_YARN_HOME}/bin/yarn" \
+ hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \
--daemon start \
--workers \
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDFS-12732. Correct spellings of
ramdomly to randomly in log. Contributed by hu xiaodong.
Posted by su...@apache.org.
HDFS-12732. Correct spellings of ramdomly to randomly in log. Contributed by hu xiaodong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a3566e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a3566e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a3566e1
Branch: refs/heads/YARN-5881
Commit: 3a3566e1d1ab5f78cfb734796b41802fe039196d
Parents: f2df6b8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Nov 9 15:13:21 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Nov 9 15:14:46 2017 +0900
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3566e1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index a479397..b925feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -614,7 +614,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to choose from local rack (location = " + localRack
- + "); the second replica is not found, retry choosing ramdomly", e);
+ + "); the second replica is not found, retry choosing randomly", e);
}
//the second replica is not found, randomly choose one from the network
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
@@ -636,7 +636,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
} catch(NotEnoughReplicasException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to choose from the next rack (location = " + nextRack
- + "), retry choosing ramdomly", e);
+ + "), retry choosing randomly", e);
}
//otherwise randomly choose one from the network
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: Merge branch
'yarn-native-services' into trunk
Posted by su...@apache.org.
Merge branch 'yarn-native-services' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbc632d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbc632d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbc632d9
Branch: refs/heads/YARN-5881
Commit: cbc632d9abf08c56a7fc02be51b2718af30bad28
Parents: dcd99c4 a55d073
Author: Jian He <ji...@apache.org>
Authored: Mon Nov 6 14:02:19 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 14:02:19 2017 -0800
----------------------------------------------------------------------
LICENSE.txt | 1 +
NOTICE.txt | 10 +
.../resources/assemblies/hadoop-yarn-dist.xml | 25 +
hadoop-project/pom.xml | 44 +-
hadoop-project/src/site/site.xml | 10 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 18 +-
.../hadoop-yarn/conf/yarn-env.sh | 18 +
.../dev-support/findbugs-exclude.xml | 16 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +
.../dev-support/findbugs-exclude.xml | 20 +
.../hadoop-yarn-services-api/pom.xml | 130 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 298 +++
.../yarn/service/webapp/ApiServerWebApp.java | 161 ++
.../definition/YARN-Services-Examples.md | 245 +++
...RN-Simplified-V1-API-Layer-For-Services.yaml | 471 +++++
.../src/main/resources/log4j-server.properties | 76 +
.../src/main/resources/webapps/api-server/app | 16 +
.../src/main/webapp/WEB-INF/web.xml | 36 +
.../hadoop/yarn/service/ServiceClientTest.java | 107 ++
.../hadoop/yarn/service/TestApiServer.java | 366 ++++
.../dev-support/findbugs-exclude.xml | 48 +
.../conf/yarnservice-log4j.properties | 62 +
.../examples/httpd-no-dns/httpd-no-dns.json | 62 +
.../httpd-no-dns/httpd-proxy-no-dns.conf | 24 +
.../examples/httpd/httpd-proxy.conf | 24 +
.../examples/httpd/httpd.json | 55 +
.../examples/sleeper/sleeper.json | 15 +
.../hadoop-yarn-services-core/pom.xml | 255 +++
.../hadoop/yarn/service/ClientAMProtocol.java | 40 +
.../hadoop/yarn/service/ClientAMService.java | 132 ++
.../yarn/service/ContainerFailureTracker.java | 89 +
.../hadoop/yarn/service/ServiceContext.java | 41 +
.../hadoop/yarn/service/ServiceMaster.java | 169 ++
.../hadoop/yarn/service/ServiceMetrics.java | 94 +
.../hadoop/yarn/service/ServiceScheduler.java | 691 +++++++
.../yarn/service/api/ServiceApiConstants.java | 74 +
.../yarn/service/api/records/Artifact.java | 168 ++
.../yarn/service/api/records/BaseResource.java | 52 +
.../yarn/service/api/records/Component.java | 430 +++++
.../service/api/records/ComponentState.java | 30 +
.../yarn/service/api/records/ConfigFile.java | 233 +++
.../yarn/service/api/records/ConfigFormat.java | 67 +
.../yarn/service/api/records/Configuration.java | 225 +++
.../yarn/service/api/records/Container.java | 298 +++
.../service/api/records/ContainerState.java | 30 +
.../hadoop/yarn/service/api/records/Error.java | 129 ++
.../service/api/records/PlacementPolicy.java | 102 +
.../service/api/records/ReadinessCheck.java | 183 ++
.../yarn/service/api/records/Resource.java | 161 ++
.../yarn/service/api/records/Service.java | 390 ++++
.../yarn/service/api/records/ServiceState.java | 33 +
.../yarn/service/api/records/ServiceStatus.java | 148 ++
.../yarn/service/client/ClientAMProxy.java | 57 +
.../yarn/service/client/ServiceClient.java | 960 ++++++++++
.../yarn/service/component/Component.java | 584 ++++++
.../yarn/service/component/ComponentEvent.java | 83 +
.../service/component/ComponentEventType.java | 27 +
.../yarn/service/component/ComponentState.java | 25 +
.../component/instance/ComponentInstance.java | 549 ++++++
.../instance/ComponentInstanceEvent.java | 58 +
.../instance/ComponentInstanceEventType.java | 26 +
.../component/instance/ComponentInstanceId.java | 91 +
.../instance/ComponentInstanceState.java | 26 +
.../yarn/service/conf/RestApiConstants.java | 39 +
.../yarn/service/conf/SliderExitCodes.java | 88 +
.../yarn/service/conf/YarnServiceConf.java | 113 ++
.../yarn/service/conf/YarnServiceConstants.java | 92 +
.../containerlaunch/AbstractLauncher.java | 271 +++
.../containerlaunch/ClasspathConstructor.java | 172 ++
.../containerlaunch/CommandLineBuilder.java | 86 +
.../containerlaunch/ContainerLaunchService.java | 101 +
.../containerlaunch/CredentialUtils.java | 319 ++++
.../containerlaunch/JavaCommandLineBuilder.java | 181 ++
.../exceptions/BadClusterStateException.java | 36 +
.../BadCommandArgumentsException.java | 30 +
.../service/exceptions/BadConfigException.java | 39 +
.../yarn/service/exceptions/ErrorStrings.java | 42 +
.../service/exceptions/ExitCodeProvider.java | 32 +
.../service/exceptions/LauncherExitCodes.java | 196 ++
.../exceptions/RestApiErrorMessages.java | 92 +
.../exceptions/ServiceLaunchException.java | 73 +
.../service/exceptions/SliderException.java | 66 +
.../yarn/service/exceptions/UsageException.java | 34 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 +
.../impl/pb/service/ClientAMProtocolPB.java | 29 +
.../service/ClientAMProtocolPBServiceImpl.java | 70 +
.../yarn/service/monitor/ServiceMonitor.java | 147 ++
.../yarn/service/monitor/probe/HttpProbe.java | 110 ++
.../service/monitor/probe/LogEntryBuilder.java | 76 +
.../yarn/service/monitor/probe/MonitorKeys.java | 66 +
.../service/monitor/probe/MonitorUtils.java | 84 +
.../yarn/service/monitor/probe/PortProbe.java | 98 +
.../yarn/service/monitor/probe/Probe.java | 100 +
.../yarn/service/monitor/probe/ProbeStatus.java | 160 ++
.../provider/AbstractClientProvider.java | 129 ++
.../provider/AbstractProviderService.java | 113 ++
.../yarn/service/provider/ProviderFactory.java | 76 +
.../yarn/service/provider/ProviderService.java | 39 +
.../yarn/service/provider/ProviderUtils.java | 408 ++++
.../defaultImpl/DefaultClientProvider.java | 46 +
.../defaultImpl/DefaultProviderFactory.java | 51 +
.../defaultImpl/DefaultProviderService.java | 36 +
.../provider/docker/DockerClientProvider.java | 53 +
.../service/provider/docker/DockerKeys.java | 30 +
.../provider/docker/DockerProviderFactory.java | 52 +
.../provider/docker/DockerProviderService.java | 57 +
.../provider/tarball/TarballClientProvider.java | 65 +
.../tarball/TarballProviderFactory.java | 52 +
.../tarball/TarballProviderService.java | 48 +
.../registry/CustomRegistryConstants.java | 57 +
.../registry/YarnRegistryViewForProviders.java | 249 +++
.../timelineservice/ServiceMetricsSink.java | 102 +
.../ServiceTimelineEntityType.java | 39 +
.../timelineservice/ServiceTimelineEvent.java | 36 +
.../ServiceTimelineMetricsConstants.java | 92 +
.../ServiceTimelinePublisher.java | 381 ++++
.../service/timelineservice/package-info.java | 27 +
.../utils/ApplicationReportSerDeser.java | 56 +
.../service/utils/ClientRegistryBinder.java | 201 ++
.../hadoop/yarn/service/utils/Comparators.java | 62 +
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 +
.../yarn/service/utils/CoreFileSystem.java | 521 ++++++
.../hadoop/yarn/service/utils/Duration.java | 109 ++
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++
.../yarn/service/utils/PatternValidator.java | 58 +
.../hadoop/yarn/service/utils/PortScanner.java | 113 ++
.../service/utils/PublishedConfiguration.java | 196 ++
.../utils/PublishedConfigurationOutputter.java | 212 +++
.../utils/SerializedApplicationReport.java | 98 +
.../yarn/service/utils/ServiceApiUtil.java | 401 ++++
.../service/utils/ServiceRegistryUtils.java | 71 +
.../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++
.../yarn/service/utils/SliderFileSystem.java | 51 +
.../yarn/service/utils/ZookeeperUtils.java | 146 ++
.../src/main/proto/ClientAMProtocol.proto | 56 +
.../hadoop/yarn/service/MockServiceAM.java | 221 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 262 +++
.../hadoop/yarn/service/TestServiceApiUtil.java | 480 +++++
.../yarn/service/TestYarnNativeServices.java | 403 ++++
.../client/TestBuildExternalComponents.java | 119 ++
.../yarn/service/client/TestServiceCLI.java | 127 ++
.../yarn/service/conf/ExampleAppJson.java | 65 +
.../yarn/service/conf/TestAppJsonResolve.java | 200 ++
.../service/conf/TestLoadExampleAppJson.java | 71 +
.../service/conf/TestValidateServiceNames.java | 125 ++
.../service/monitor/TestServiceMonitor.java | 104 ++
.../providers/TestAbstractClientProvider.java | 118 ++
.../service/providers/TestProviderFactory.java | 76 +
.../TestServiceTimelinePublisher.java | 291 +++
.../src/test/resources/example-app.json | 15 +
.../service/conf/examples/app-override.json | 76 +
.../hadoop/yarn/service/conf/examples/app.json | 50 +
.../yarn/service/conf/examples/default.json | 16 +
.../yarn/service/conf/examples/external0.json | 15 +
.../yarn/service/conf/examples/external1.json | 30 +
.../yarn/service/conf/examples/external2.json | 22 +
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop-yarn-services/pom.xml | 38 +
.../hadoop-yarn-applications/pom.xml | 2 +
.../hadoop/yarn/client/api/AppAdminClient.java | 222 +++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 318 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 158 +-
.../yarn/security/ContainerTokenIdentifier.java | 20 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 36 +
.../src/main/proto/yarn_security_token.proto | 1 +
.../src/main/resources/yarn-default.xml | 8 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 78 +-
.../registry/client/api/DNSOperations.java | 60 +
.../client/api/DNSOperationsFactory.java | 78 +
.../registry/client/api/RegistryConstants.java | 136 +-
.../client/api/RegistryOperationsFactory.java | 8 +
.../client/binding/RegistryTypeUtils.java | 3 +-
.../registry/client/impl/zk/CuratorService.java | 271 ++-
.../registry/client/impl/zk/ListenerHandle.java | 25 +
.../registry/client/impl/zk/PathListener.java | 30 +
.../impl/zk/RegistryOperationsService.java | 4 +-
.../types/yarn/YarnRegistryAttributes.java | 16 +-
.../dns/ApplicationServiceRecordProcessor.java | 366 ++++
.../server/dns/BaseServiceRecordProcessor.java | 468 +++++
.../dns/ContainerServiceRecordProcessor.java | 283 +++
.../dns/PrivilegedRegistryDNSStarter.java | 80 +
.../server/dns/RecordCreatorFactory.java | 275 +++
.../hadoop/registry/server/dns/RegistryDNS.java | 1756 ++++++++++++++++++
.../registry/server/dns/RegistryDNSServer.java | 267 +++
.../registry/server/dns/ReverseZoneUtils.java | 171 ++
.../registry/server/dns/SecureableZone.java | 151 ++
.../server/dns/ServiceRecordProcessor.java | 53 +
.../registry/server/dns/ZoneSelector.java | 33 +
.../registry/server/dns/package-info.java | 26 +
.../registry/server/dns/TestRegistryDNS.java | 627 +++++++
.../server/dns/TestReverseZoneUtils.java | 89 +
.../server/dns/TestSecureRegistryDNS.java | 44 +
.../test/resources/0.17.172.in-addr.arpa.zone | 36 +
.../src/test/resources/test.private | 32 +
.../api/protocolrecords/NMContainerStatus.java | 18 +-
.../impl/pb/NMContainerStatusPBImpl.java | 15 +-
.../OpportunisticContainerAllocator.java | 2 +-
.../yarn_server_common_service_protos.proto | 1 +
.../containermanager/ContainerManagerImpl.java | 5 +-
.../container/ContainerImpl.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 8 +
.../blacklist/SimpleBlacklistManager.java | 8 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 35 +-
.../scheduler/AbstractYarnScheduler.java | 1 +
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../security/RMContainerTokenSecretManager.java | 36 +-
.../server/resourcemanager/webapp/RMWebApp.java | 19 +
.../server/resourcemanager/TestRMRestart.java | 2 +-
.../TestResourceTrackerService.java | 6 +-
.../capacity/TestContainerAllocation.java | 5 +-
.../src/site/markdown/YarnCommands.md | 33 +-
.../src/site/markdown/registry/index.md | 1 +
.../markdown/registry/registry-configuration.md | 4 +-
.../src/site/markdown/yarn-service/Concepts.md | 46 +
.../markdown/yarn-service/Configurations.md | 165 ++
.../src/site/markdown/yarn-service/Examples.md | 161 ++
.../src/site/markdown/yarn-service/Overview.md | 61 +
.../site/markdown/yarn-service/QuickStart.md | 217 +++
.../site/markdown/yarn-service/RegistryDNS.md | 200 ++
.../markdown/yarn-service/ServiceDiscovery.md | 136 ++
.../markdown/yarn-service/YarnServiceAPI.md | 598 ++++++
.../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes
.../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes
.../resources/images/dns_record_removal.jpeg | Bin 0 -> 58041 bytes
.../main/webapp/app/adapters/restabstract.js | 50 +
.../app/adapters/yarn-component-instance.js | 32 +
.../app/adapters/yarn-service-component.js | 32 +
.../webapp/app/adapters/yarn-service-info.js | 32 +
.../main/webapp/app/adapters/yarn-servicedef.js | 44 +
.../webapp/app/components/breadcrumb-bar.js | 1 +
.../webapp/app/components/confirm-dialog.js | 31 +
.../webapp/app/components/deploy-service.js | 167 ++
.../app/components/fileconfig-viewer-dialog.js | 36 +
.../main/webapp/app/components/info-tooltip.js | 44 +
.../main/webapp/app/components/metrics-table.js | 24 +
.../app/components/service-component-table.js | 56 +
.../app/components/service-config-table.js | 89 +
.../app/components/service-fileconfig-table.js | 112 ++
.../main/webapp/app/components/upload-config.js | 54 +
.../app/controllers/yarn-app/components.js | 59 +
.../webapp/app/controllers/yarn-app/configs.js | 24 +
.../webapp/app/controllers/yarn-app/info.js | 60 +
.../app/controllers/yarn-component-instance.js | 59 +
.../controllers/yarn-component-instance/info.js | 25 +
.../app/controllers/yarn-component-instances.js | 59 +
.../yarn-component-instances/configs.js | 25 +
.../yarn-component-instances/info.js | 62 +
.../app/controllers/yarn-deploy-service.js | 69 +
.../webapp/app/helpers/check-availability.js | 28 +
.../app/models/yarn-component-instance.js | 51 +
.../webapp/app/models/yarn-service-component.js | 46 +
.../main/webapp/app/models/yarn-service-info.js | 57 +
.../main/webapp/app/models/yarn-servicedef.js | 278 +++
.../src/main/webapp/app/router.js | 10 +
.../webapp/app/routes/yarn-app/components.js | 49 +
.../main/webapp/app/routes/yarn-app/configs.js | 52 +
.../src/main/webapp/app/routes/yarn-app/info.js | 12 +-
.../app/routes/yarn-component-instance.js | 29 +
.../app/routes/yarn-component-instance/info.js | 45 +
.../app/routes/yarn-component-instances.js | 29 +
.../routes/yarn-component-instances/configs.js | 44 +
.../app/routes/yarn-component-instances/info.js | 53 +
.../webapp/app/routes/yarn-deploy-service.js | 27 +
.../src/main/webapp/app/routes/yarn-services.js | 2 +-
.../app/serializers/yarn-component-instance.js | 72 +
.../app/serializers/yarn-service-component.js | 77 +
.../webapp/app/serializers/yarn-service-info.js | 87 +
.../src/main/webapp/app/services/hosts.js | 4 +
.../src/main/webapp/app/styles/app.css | 176 ++
.../main/webapp/app/templates/application.hbs | 2 +
.../app/templates/components/breadcrumb-bar.hbs | 4 +-
.../app/templates/components/confirm-dialog.hbs | 37 +
.../app/templates/components/deploy-service.hbs | 157 ++
.../components/fileconfig-viewer-dialog.hbs | 53 +
.../app/templates/components/info-tooltip.hbs | 20 +
.../app/templates/components/metrics-table.hbs | 55 +
.../components/service-component-table.hbs | 113 ++
.../components/service-config-table.hbs | 130 ++
.../components/service-fileconfig-table.hbs | 152 ++
.../app/templates/components/upload-config.hbs | 44 +
.../src/main/webapp/app/templates/yarn-app.hbs | 8 +
.../app/templates/yarn-app/components.hbs | 23 +
.../webapp/app/templates/yarn-app/configs.hbs | 57 +
.../main/webapp/app/templates/yarn-app/info.hbs | 111 +-
.../app/templates/yarn-component-instance.hbs | 43 +
.../templates/yarn-component-instance/info.hbs | 77 +
.../app/templates/yarn-component-instances.hbs | 46 +
.../yarn-component-instances/configs.hbs | 53 +
.../templates/yarn-component-instances/info.hbs | 28 +
.../yarn-component-instances/loading.hbs | 23 +
.../app/templates/yarn-deploy-service.hbs | 33 +
.../main/webapp/app/templates/yarn-services.hbs | 4 +
.../src/main/webapp/app/utils/info-seeder.js | 26 +
.../src/main/webapp/config/default-config.js | 6 +-
.../components/confirm-dialog-test.js | 43 +
.../components/deploy-service-test.js | 43 +
.../components/fileconfig-viewer-dialog-test.js | 43 +
.../integration/components/info-tooltip-test.js | 43 +
.../components/metrics-table-test.js | 43 +
.../components/service-component-table-test.js | 43 +
.../components/service-config-table-test.js | 43 +
.../components/service-fileconfig-table-test.js | 43 +
.../components/upload-config-test.js | 43 +
.../adapters/yarn-component-instance-test.js | 30 +
.../adapters/yarn-service-component-test.js | 30 +
.../unit/adapters/yarn-service-info-test.js | 30 +
.../tests/unit/adapters/yarn-servicedef-test.js | 30 +
.../controllers/yarn-app/components-test.js | 30 +
.../unit/controllers/yarn-app/configs-test.js | 30 +
.../controllers/yarn-component-instance-test.js | 30 +
.../yarn-component-instance/info-test.js | 30 +
.../yarn-component-instances-test.js | 30 +
.../yarn-component-instances/configs-test.js | 30 +
.../yarn-component-instances/info-test.js | 30 +
.../controllers/yarn-deploy-service-test.js | 30 +
.../unit/helpers/check-availability-test.js | 28 +
.../unit/models/yarn-component-instance-test.js | 30 +
.../unit/models/yarn-service-component-test.js | 30 +
.../tests/unit/models/yarn-service-info-test.js | 30 +
.../tests/unit/models/yarn-servicedef-test.js | 30 +
.../unit/routes/yarn-app/components-test.js | 29 +
.../tests/unit/routes/yarn-app/configs-test.js | 29 +
.../unit/routes/yarn-component-instance-test.js | 29 +
.../routes/yarn-component-instance/info-test.js | 29 +
.../routes/yarn-component-instances-test.js | 29 +
.../yarn-component-instances/configs-test.js | 29 +
.../yarn-component-instances/info-test.js | 29 +
.../unit/routes/yarn-deploy-service-test.js | 29 +
.../serializers/yarn-component-instance-test.js | 33 +
.../serializers/yarn-service-component-test.js | 33 +
.../unit/serializers/yarn-service-info-test.js | 33 +
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
hadoop-yarn-project/pom.xml | 4 +
334 files changed, 33539 insertions(+), 320 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HADOOP-15025. Ensure singleton for
ResourceEstimatorService. (Rui Li via Subru).
Posted by su...@apache.org.
HADOOP-15025. Ensure singleton for ResourceEstimatorService. (Rui Li via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2df6b89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2df6b89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2df6b89
Branch: refs/heads/YARN-5881
Commit: f2df6b8983aace73ad27934bd9f7f4d766e0b25f
Parents: 49b4c0b
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Nov 8 18:07:12 2017 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Nov 8 18:07:12 2017 -0800
----------------------------------------------------------------------
.../service/ResourceEstimatorService.java | 5 ++--
.../service/TestResourceEstimatorService.java | 25 +-------------------
2 files changed, 4 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2df6b89/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
index 0e0e094..5d3aea4 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/service/ResourceEstimatorService.java
@@ -34,6 +34,7 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
+import com.sun.jersey.spi.resource.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.resourceestimator.common.api.RecurrenceId;
import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline;
@@ -56,13 +57,13 @@ import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-import com.google.inject.Singleton;
/**
* Resource Estimator Service which provides a set of REST APIs for users to
* use the estimation service.
*/
-@Singleton @Path("/resourceestimator") public class ResourceEstimatorService {
+@Singleton
+@Path("/resourceestimator") public class ResourceEstimatorService {
private static final Logger LOGGER =
LoggerFactory.getLogger(ResourceEstimatorService.class);
private final SkylineStore skylineStore;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2df6b89/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
index 91a486e..785641c 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/test/java/org/apache/hadoop/resourceestimator/service/TestResourceEstimatorService.java
@@ -37,18 +37,12 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.JerseyTest;
-import com.sun.jersey.test.framework.WebAppDescriptor;
/**
* Test ResourceEstimatorService.
@@ -70,29 +64,12 @@ public class TestResourceEstimatorService extends JerseyTest {
private long containerMemAlloc;
private int containerCPUAlloc;
- private static class WebServletModule extends ServletModule {
- @Override protected void configureServlets() {
- bind(ResourceEstimatorService.class);
- serve("/*").with(GuiceContainer.class);
- }
- }
-
- static {
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
- }
-
public TestResourceEstimatorService() {
- super(new WebAppDescriptor.Builder(
- "org.apache.hadoop.resourceestimator.service")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class).build());
+ super("org.apache.hadoop.resourceestimator.service");
}
@Before @Override public void setUp() throws Exception {
super.setUp();
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
containerMemAlloc = 1024;
containerCPUAlloc = 1;
containerSpec = Resource.newInstance(containerMemAlloc, containerCPUAlloc);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: MAPREDUCE-6997. Moving logging APIs over to slf4j in hadoop-mapreduce-client-hs. Contributed by Gergely Novák.
Posted by su...@apache.org.
MAPREDUCE-6997. Moving logging APIs over to slf4j in hadoop-mapreduce-client-hs. Contributed by Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffee10b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffee10b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffee10b6
Branch: refs/heads/YARN-5881
Commit: ffee10b68ef1f2d75c9d0df9140c2a605f826724
Parents: 410d031
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Nov 8 19:21:43 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Nov 8 19:21:43 2017 +0900
----------------------------------------------------------------------
.../mapreduce/v2/hs/CachedHistoryStorage.java | 7 ++++---
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 6 +++---
.../hadoop/mapreduce/v2/hs/HSAuditLogger.java | 7 ++++---
.../hadoop/mapreduce/v2/hs/HSProxies.java | 6 +++---
.../mapreduce/v2/hs/HistoryClientService.java | 7 ++++---
.../mapreduce/v2/hs/HistoryFileManager.java | 10 +++++----
...istoryServerFileSystemStateStoreService.java | 20 +++++++++---------
.../HistoryServerLeveldbStateStoreService.java | 22 ++++++++++----------
.../v2/hs/JHSDelegationTokenSecretManager.java | 6 +++---
.../hadoop/mapreduce/v2/hs/JobHistory.java | 6 +++---
.../mapreduce/v2/hs/JobHistoryServer.java | 9 ++++----
.../hadoop/mapreduce/v2/hs/PartialJob.java | 6 +++---
.../mapreduce/v2/hs/server/HSAdminServer.java | 7 ++++---
.../mapreduce/v2/hs/TestJobHistoryEvents.java | 7 ++++---
.../mapreduce/v2/hs/TestJobHistoryParsing.java | 7 ++++---
.../mapreduce/v2/hs/webapp/TestHSWebApp.java | 6 +++---
16 files changed, 74 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
index c59d17f..b001ae4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
@@ -30,8 +30,6 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.Weigher;
import com.google.common.util.concurrent.UncheckedExecutionException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
@@ -45,13 +43,16 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Manages an in memory cache of parsed Job History files.
*/
public class CachedHistoryStorage extends AbstractService implements
HistoryStorage {
- private static final Log LOG = LogFactory.getLog(CachedHistoryStorage.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CachedHistoryStorage.class);
private LoadingCache<JobId, Job> loadedJobCache = null;
private int loadedJobCacheSize;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index 5afb645..a4e75f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -31,8 +31,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
@@ -63,6 +61,8 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -75,7 +75,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
// so we count it as 0
private static final int UNDEFINED_VALUE = -1;
- static final Log LOG = LogFactory.getLog(CompletedJob.class);
+ private static final Logger LOG = LoggerFactory.getLogger(CompletedJob.class);
private final Configuration conf;
private final JobId jobId; //Can be picked from JobInfo with a conversion.
private final String user; //Can be picked up from JobInfo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSAuditLogger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSAuditLogger.java
index 5d851a5..6bf9794 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSAuditLogger.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSAuditLogger.java
@@ -20,14 +20,15 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.net.InetAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.ipc.Server;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
public class HSAuditLogger {
- private static final Log LOG = LogFactory.getLog(HSAuditLogger.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HSAuditLogger.class);
enum Keys {
USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java
index 65fe950..3e238cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -38,11 +36,13 @@ import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
public class HSProxies {
- private static final Log LOG = LogFactory.getLog(HSProxies.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HSProxies.class);
@SuppressWarnings("unchecked")
public static <T> T createProxy(Configuration conf, InetSocketAddress hsaddr,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 487794c..b0bf41b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -26,8 +26,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -90,6 +88,8 @@ import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This module is responsible for talking to the
@@ -98,7 +98,8 @@ import com.google.common.annotations.VisibleForTesting;
*/
public class HistoryClientService extends AbstractService {
- private static final Log LOG = LogFactory.getLog(HistoryClientService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HistoryClientService.class);
private HSClientProtocol protocolHandler;
private Server server;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index c7d6d19..61c750e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -40,8 +40,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -76,6 +74,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class provides a way to interact with history files in a thread safe
@@ -84,8 +84,10 @@ import org.apache.hadoop.yarn.util.SystemClock;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class HistoryFileManager extends AbstractService {
- private static final Log LOG = LogFactory.getLog(HistoryFileManager.class);
- private static final Log SUMMARY_LOG = LogFactory.getLog(JobSummary.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HistoryFileManager.class);
+ private static final Logger SUMMARY_LOG =
+ LoggerFactory.getLogger(JobSummary.class);
private enum HistoryInfoState {
IN_INTERMEDIATE, IN_DONE, DELETED, MOVE_FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
index 47d6583..7601602 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
@@ -27,8 +27,6 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -44,6 +42,8 @@ import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
@Unstable
@@ -54,8 +54,8 @@ import org.apache.hadoop.util.Shell;
public class HistoryServerFileSystemStateStoreService
extends HistoryServerStateStoreService {
- public static final Log LOG =
- LogFactory.getLog(HistoryServerFileSystemStateStoreService.class);
+ public static final Logger LOG =
+ LoggerFactory.getLogger(HistoryServerFileSystemStateStoreService.class);
private static final String ROOT_STATE_DIR_NAME = "HistoryServerState";
private static final String TOKEN_STATE_DIR_NAME = "tokens";
@@ -192,7 +192,7 @@ public class HistoryServerFileSystemStateStoreService
dataStream.close();
dataStream = null;
} finally {
- IOUtils.cleanup(LOG, dataStream);
+ IOUtils.cleanupWithLogger(LOG, dataStream);
}
createNewFile(keyPath, memStream.toByteArray());
@@ -265,7 +265,7 @@ public class HistoryServerFileSystemStateStoreService
out.close();
out = null;
} finally {
- IOUtils.cleanup(LOG, out);
+ IOUtils.cleanupWithLogger(LOG, out);
}
} catch (IOException e) {
fs.delete(file, false);
@@ -279,7 +279,7 @@ public class HistoryServerFileSystemStateStoreService
try {
in.readFully(data);
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
return data;
}
@@ -306,7 +306,7 @@ public class HistoryServerFileSystemStateStoreService
dataStream.close();
dataStream = null;
} finally {
- IOUtils.cleanup(LOG, dataStream);
+ IOUtils.cleanupWithLogger(LOG, dataStream);
}
return memStream.toByteArray();
}
@@ -320,7 +320,7 @@ public class HistoryServerFileSystemStateStoreService
try {
key.readFields(in);
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
state.tokenMasterKeyState.add(key);
}
@@ -349,7 +349,7 @@ public class HistoryServerFileSystemStateStoreService
tokenId.readFields(in);
renewDate = in.readLong();
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
state.tokenState.put(tokenId, renewDate);
return tokenId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
index 16366b1..c8741aa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java
@@ -29,8 +29,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Map.Entry;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -47,8 +45,9 @@ import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HistoryServerLeveldbStateStoreService extends
HistoryServerStateStoreService {
@@ -63,8 +62,8 @@ public class HistoryServerLeveldbStateStoreService extends
private DB db;
- public static final Log LOG =
- LogFactory.getLog(HistoryServerLeveldbStateStoreService.class);
+ public static final Logger LOG =
+ LoggerFactory.getLogger(HistoryServerLeveldbStateStoreService.class);
@Override
protected void initStorage(Configuration conf) throws IOException {
@@ -158,7 +157,7 @@ public class HistoryServerLeveldbStateStoreService extends
try {
key.readFields(in);
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
state.tokenMasterKeyState.add(key);
}
@@ -204,7 +203,7 @@ public class HistoryServerLeveldbStateStoreService extends
tokenId.readFields(in);
renewDate = in.readLong();
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
state.tokenState.put(tokenId, renewDate);
}
@@ -224,7 +223,7 @@ public class HistoryServerLeveldbStateStoreService extends
dataStream.close();
dataStream = null;
} finally {
- IOUtils.cleanup(LOG, dataStream);
+ IOUtils.cleanupWithLogger(LOG, dataStream);
}
String dbKey = getTokenDatabaseKey(tokenId);
@@ -270,7 +269,7 @@ public class HistoryServerLeveldbStateStoreService extends
dataStream.close();
dataStream = null;
} finally {
- IOUtils.cleanup(LOG, dataStream);
+ IOUtils.cleanupWithLogger(LOG, dataStream);
}
String dbKey = getTokenMasterKeyDatabaseKey(masterKey);
@@ -368,8 +367,9 @@ public class HistoryServerLeveldbStateStoreService extends
}
}
- private static class LeveldbLogger implements Logger {
- private static final Log LOG = LogFactory.getLog(LeveldbLogger.class);
+ private static class LeveldbLogger implements org.iq80.leveldb.Logger {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LeveldbLogger.class);
@Override
public void log(String message) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JHSDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JHSDelegationTokenSecretManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JHSDelegationTokenSecretManager.java
index 98d13a4..2500f68 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JHSDelegationTokenSecretManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JHSDelegationTokenSecretManager.java
@@ -21,14 +21,14 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
import java.util.Map.Entry;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A MapReduce specific delegation token secret manager.
@@ -40,7 +40,7 @@ import org.apache.hadoop.security.token.delegation.DelegationKey;
public class JHSDelegationTokenSecretManager
extends AbstractDelegationTokenSecretManager<MRDelegationTokenIdentifier> {
- private static final Log LOG = LogFactory.getLog(
+ private static final Logger LOG = LoggerFactory.getLogger(
JHSDelegationTokenSecretManager.class);
private HistoryServerStateStoreService store;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
index 2671df4..2d263dd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
@@ -27,8 +27,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -56,12 +54,14 @@ import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Loads and manages the Job history cache.
*/
public class JobHistory extends AbstractService implements HistoryContext {
- private static final Log LOG = LogFactory.getLog(JobHistory.class);
+ private static final Logger LOG = LoggerFactory.getLogger(JobHistory.class);
public static final Pattern CONF_FILENAME_REGEX = Pattern.compile("("
+ JobID.JOBID_REGEX + ")_conf.xml(?:\\.[0-9]+\\.old)?");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
index 092679f..9ea140f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
@@ -47,6 +45,8 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/******************************************************************
* {@link JobHistoryServer} is responsible for servicing all job history
@@ -62,7 +62,8 @@ public class JobHistoryServer extends CompositeService {
public static final long historyServerTimeStamp = System.currentTimeMillis();
- private static final Log LOG = LogFactory.getLog(JobHistoryServer.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(JobHistoryServer.class);
protected HistoryContext historyContext;
private HistoryClientService clientService;
private JobHistory jobHistoryService;
@@ -225,7 +226,7 @@ public class JobHistoryServer extends CompositeService {
jobHistoryServer.init(conf);
jobHistoryServer.start();
} catch (Throwable t) {
- LOG.fatal("Error starting JobHistoryServer", t);
+ LOG.error("Error starting JobHistoryServer", t);
ExitUtil.terminate(-1, "Error starting JobHistoryServer");
}
return jobHistoryServer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
index b14f0c3..26755d7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@@ -41,10 +39,12 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
- private static final Log LOG = LogFactory.getLog(PartialJob.class);
+ private static final Logger LOG = LoggerFactory.getLogger(PartialJob.class);
private JobIndexInfo jobIndexInfo = null;
private JobId jobId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 729af0a..1097b7e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -22,8 +22,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -54,11 +52,14 @@ import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolServer
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Private
public class HSAdminServer extends AbstractService implements HSAdminProtocol {
- private static final Log LOG = LogFactory.getLog(HSAdminServer.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HSAdminServer.class);
private AccessControlList adminAcl;
private AggregatedLogDeletionService aggLogDelService = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
index 3ce0f74..e7e912f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
@@ -22,8 +22,6 @@ import java.util.Map;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
@@ -44,9 +42,12 @@ import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestJobHistoryEvents {
- private static final Log LOG = LogFactory.getLog(TestJobHistoryEvents.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestJobHistoryEvents.class);
@Test
public void testHistoryEvents() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
index e881b37..83e35fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
@@ -38,8 +38,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Assert;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
@@ -96,9 +94,12 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestJobHistoryParsing {
- private static final Log LOG = LogFactory.getLog(TestJobHistoryParsing.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestJobHistoryParsing.class);
private static final String RACK_NAME = "/MyRackName";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffee10b6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
index d4f0d70..287c646 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHSWebApp.java
@@ -35,8 +35,6 @@ import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
@@ -54,9 +52,11 @@ import org.junit.Test;
import com.google.inject.AbstractModule;
import com.google.inject.Injector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestHSWebApp {
- private static final Log LOG = LogFactory.getLog(TestHSWebApp.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestHSWebApp.class);
@Test public void testAppControllerIndex() {
MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-7360.
TestRM.testNMTokenSentForNormalContainer() should be scheduler agnostic.
Posted by su...@apache.org.
YARN-7360. TestRM.testNMTokenSentForNormalContainer() should be scheduler agnostic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f214dc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f214dc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f214dc4
Branch: refs/heads/YARN-5881
Commit: 8f214dc4f8423250947a3f0027f70b9ab402ab62
Parents: cbc632d
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Nov 6 15:45:37 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Mon Nov 6 15:45:37 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/TestRM.java | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f214dc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
index f912f68..3679319 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import com.google.common.base.Supplier;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.event.DrainDispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.junit.Before;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.doNothing;
@@ -73,7 +73,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.log4j.Level;
@@ -205,8 +204,6 @@ public class TestRM extends ParameterizedSchedulerTestBase {
// corresponding NM Token.
@Test (timeout = 20000)
public void testNMTokenSentForNormalContainer() throws Exception {
- conf.set(YarnConfiguration.RM_SCHEDULER,
- CapacityScheduler.class.getCanonicalName());
MockRM rm = new MockRM(conf);
rm.start();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
@@ -215,8 +212,10 @@ public class TestRM extends ParameterizedSchedulerTestBase {
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- cs.getApplicationAttempt(attempt.getAppAttemptId()).getNewContainerId();
+ AbstractYarnScheduler scheduler = (AbstractYarnScheduler)
+ rm.getResourceScheduler();
+ scheduler.getApplicationAttempt(attempt.getAppAttemptId()).
+ getNewContainerId();
MockAM am = MockRM.launchAM(app, rm, nm1);
// am container Id not equal to 1.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-7453. Fix issue where RM
fails to switch to active after first successful start. (Rohith Sharma K S
via asuresh)
Posted by su...@apache.org.
YARN-7453. Fix issue where RM fails to switch to active after first successful start. (Rohith Sharma K S via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9c70b0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9c70b0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9c70b0e
Branch: refs/heads/YARN-5881
Commit: a9c70b0e84dab0c41e480a0dc0cb1a22efdc64ee
Parents: e4c220e
Author: Arun Suresh <as...@apache.org>
Authored: Wed Nov 8 08:00:53 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Nov 8 08:00:53 2017 -0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/ResourceManager.java | 12 ++++++------
.../server/resourcemanager/recovery/ZKRMStateStore.java | 5 ++++-
.../scheduler/capacity/conf/ZKConfigurationStore.java | 3 ++-
3 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c70b0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 07f5a76..727bc52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -358,7 +358,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED);
if (curatorEnabled) {
- this.zkManager = getAndStartZKManager(conf);
+ this.zkManager = createAndStartZKManager(conf);
elector = new CuratorBasedElectorService(this);
} else {
elector = new ActiveStandbyElectorBasedElectorService(this);
@@ -372,11 +372,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
* @return ZooKeeper Curator manager.
* @throws IOException If it cannot create the manager.
*/
- public synchronized ZKCuratorManager getAndStartZKManager(Configuration
+ public ZKCuratorManager createAndStartZKManager(Configuration
config) throws IOException {
- if (this.zkManager != null) {
- return zkManager;
- }
ZKCuratorManager manager = new ZKCuratorManager(config);
// Get authentication
@@ -396,7 +393,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
manager.start(authInfos);
- this.zkManager = manager;
+ return manager;
+ }
+
+ public ZKCuratorManager getZKManager() {
return zkManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c70b0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 5d3ca45..36b55e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -358,7 +358,10 @@ public class ZKRMStateStore extends RMStateStore {
amrmTokenSecretManagerRoot =
getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT);
reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT);
- zkManager = resourceManager.getAndStartZKManager(conf);
+ zkManager = resourceManager.getZKManager();
+ if(zkManager==null) {
+ zkManager = resourceManager.createAndStartZKManager(conf);
+ }
delegationTokenNodeSplitIndex =
conf.getInt(YarnConfiguration.ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX,
YarnConfiguration.DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c70b0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index a0bba8c..497ebfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -81,7 +81,8 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
this.znodeParentPath =
conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
- this.zkManager = rmContext.getResourceManager().getAndStartZKManager(conf);
+ this.zkManager =
+ rmContext.getResourceManager().createAndStartZKManager(conf);
this.zkAcl = ZKCuratorManager.getZKAcls(conf);
this.zkVersionPath = getNodePath(znodeParentPath, ZK_VERSION_PATH);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-7326. Add recursion support
and configure RegistryDNS to lookup upstream. Contributed by Eric Yang
Posted by su...@apache.org.
YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/501be9b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/501be9b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/501be9b4
Branch: refs/heads/YARN-5881
Commit: 501be9b4be7d549b23cc501b26fdfdce0a7aa911
Parents: 68acd88
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Oct 25 07:37:03 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../hadoop/registry/server/dns/RegistryDNS.java | 148 ++++++++++++++++++-
.../registry/server/dns/TestRegistryDNS.java | 8 +-
2 files changed, 148 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index d7a415d..37e8429 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -37,8 +37,10 @@ import org.xbill.DNS.DNSKEYRecord;
import org.xbill.DNS.DNSSEC;
import org.xbill.DNS.DSRecord;
import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.ExtendedResolver;
import org.xbill.DNS.Flags;
import org.xbill.DNS.Header;
+import org.xbill.DNS.Lookup;
import org.xbill.DNS.Message;
import org.xbill.DNS.NSRecord;
import org.xbill.DNS.Name;
@@ -49,9 +51,12 @@ import org.xbill.DNS.RRSIGRecord;
import org.xbill.DNS.RRset;
import org.xbill.DNS.Rcode;
import org.xbill.DNS.Record;
+import org.xbill.DNS.Resolver;
+import org.xbill.DNS.ResolverConfig;
import org.xbill.DNS.SOARecord;
import org.xbill.DNS.Section;
import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.SimpleResolver;
import org.xbill.DNS.TSIG;
import org.xbill.DNS.TSIGRecord;
import org.xbill.DNS.TextParseException;
@@ -66,8 +71,11 @@ import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
import java.net.Socket;
import java.net.SocketAddress;
+import java.net.SocketException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.DatagramChannel;
import java.nio.channels.ServerSocketChannel;
@@ -78,10 +86,13 @@ import java.security.PrivateKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.RSAPrivateKeySpec;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
+import java.util.Enumeration;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
@@ -134,6 +145,16 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
private boolean channelsInitialized = false;
/**
+ * Lock to update resolver only once per request.
+ */
+ private final Object resolverUpdateLock = new Object();
+
+ /**
+ * Whether resolver update has been requested.
+ */
+ private boolean resolverUpdateRequested = true;
+
+ /**
* Construct the service.
*
* @param name service name
@@ -172,6 +193,79 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
/**
+ * Initialize registryDNS to use /etc/resolv.conf values
+ * as default resolvers.
+ */
+ private void updateDNSServer(Configuration conf) {
+ synchronized (resolverUpdateLock) {
+ if (!resolverUpdateRequested) {
+ return;
+ }
+ int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ resolverUpdateRequested = false;
+ List<InetAddress> list = new ArrayList<InetAddress>();
+ try {
+ // If resolv.conf contains the server's own IP address,
+ // and RegistryDNS handles the lookup. Local IP address
+ // must be filter out from default resolvers to prevent
+ // self recursive loop.
+ if (port != 53) {
+ // When registryDNS is not running on default port,
+ // registryDNS can utilize local DNS server as upstream lookup.
+ throw new SocketException("Bypass filtering local DNS server.");
+ }
+ Enumeration<NetworkInterface> net =
+ NetworkInterface.getNetworkInterfaces();
+ while(net.hasMoreElements()) {
+ NetworkInterface n = (NetworkInterface) net.nextElement();
+ Enumeration<InetAddress> ee = n.getInetAddresses();
+ while (ee.hasMoreElements()) {
+ InetAddress i = (InetAddress) ee.nextElement();
+ list.add(i);
+ }
+ }
+ } catch (SocketException e) {
+ }
+ ResolverConfig.refresh();
+ ExtendedResolver resolver;
+ try {
+ resolver = new ExtendedResolver();
+ } catch (UnknownHostException e) {
+ LOG.error("Can not resolve DNS servers: ", e);
+ return;
+ }
+ for (Resolver check : resolver.getResolvers()) {
+ if (check instanceof SimpleResolver) {
+ InetAddress address = ((SimpleResolver) check).getAddress()
+ .getAddress();
+ if (list.contains(address)) {
+ resolver.deleteResolver(check);
+ continue;
+ } else {
+ check.setTimeout(30);
+ }
+ } else {
+ LOG.error("Not simple resolver!!!?" + check);
+ }
+ }
+ synchronized (Lookup.class) {
+ Lookup.setDefaultResolver(resolver);
+ Lookup.setDefaultSearchPath(ResolverConfig.getCurrentConfig()
+ .searchPath());
+ }
+ StringBuilder message = new StringBuilder();
+ message.append("DNS servers: ");
+ if (ResolverConfig.getCurrentConfig().servers() != null) {
+ for (String server : ResolverConfig.getCurrentConfig()
+ .servers()) {
+ message.append(server);
+ message.append(" ");
+ }
+ }
+ LOG.info(message.toString());
+ }
+ }
+ /**
* Initializes the registry.
*
* @param conf the hadoop configuration
@@ -183,6 +277,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
// create the zone. for now create a "dummy" SOA record
try {
+ updateDNSServer(conf);
setDomainName(conf);
initializeZones(conf);
@@ -916,9 +1011,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
for (int i = 0; i < 4; i++) {
response.removeAllRecords(i);
}
- if (rcode == Rcode.SERVFAIL) {
- response.addRecord(question, Section.QUESTION);
- }
+ response.addRecord(question, Section.QUESTION);
header.setRcode(rcode);
return response.toWire();
}
@@ -975,6 +1068,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
response.getHeader().setFlag(Flags.QR);
if (query.getHeader().getFlag(Flags.RD)) {
response.getHeader().setFlag(Flags.RD);
+ response.getHeader().setFlag(Flags.RA);
}
response.addRecord(queryRecord, Section.QUESTION);
@@ -992,10 +1086,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.debug("calling addAnswer");
byte rcode = addAnswer(response, name, type, dclass, 0, flags);
- if (rcode != Rcode.NOERROR && rcode != Rcode.NXDOMAIN) {
- return errorMessage(query, rcode);
+ if (rcode != Rcode.NOERROR) {
+ rcode = remoteLookup(response, name);
+ response.getHeader().setRcode(rcode);
}
-
addAdditional(response, flags);
if (queryOPT != null) {
@@ -1009,6 +1103,45 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
/**
+ * Lookup record from upstream DNS servers.
+ */
+ private byte remoteLookup(Message response, Name name) {
+ // Forward lookup to primary DNS servers
+ Record[] answers = getRecords(name, Type.ANY);
+ try {
+ for (Record r : answers) {
+ if (r.getType() == Type.SOA) {
+ response.addRecord(r, Section.AUTHORITY);
+ } else {
+ response.addRecord(r, Section.ANSWER);
+ }
+ }
+ } catch (NullPointerException e) {
+ return Rcode.NXDOMAIN;
+ } catch (Throwable e) {
+ return Rcode.SERVFAIL;
+ }
+ return Rcode.NOERROR;
+ }
+
+ /**
+ * Requests records for the given resource name.
+ *
+ * @param name - query string
+ * @param type - type of DNS record to lookup
+ * @return DNS records
+ */
+ protected Record[] getRecords(Name name, int type) {
+ try {
+ return new Lookup(name, type).run();
+ } catch (NullPointerException |
+ ExceptionInInitializerError e) {
+ LOG.error("Fail to lookup: " + name, e);
+ }
+ return null;
+ }
+
+ /**
* Create a query to forward to the primary DNS server (if configured).
* NOTE: Experimental
*
@@ -1180,7 +1313,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
rcode = Rcode.NOTAUTH;
}
}
- LOG.info("found record? {}", sr != null && sr.isSuccessful());
+ LOG.info("found local record? {}", sr != null && sr.isSuccessful());
if (sr != null) {
if (sr.isCNAME()) {
@@ -1241,6 +1374,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
}
}
+
return rcode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index ac8d939..7c78161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -333,7 +333,7 @@ public class TestRegistryDNS extends Assert {
query.addRecord(optRecord, Section.ADDITIONAL);
byte[] responseBytes = getRegistryDNS().generateReply(query, null);
Message response = new Message(responseBytes);
- assertEquals("No answer should be returned", Rcode.NOTAUTH,
+ assertEquals("Missing record should be: ", Rcode.NXDOMAIN,
response.getRcode());
}
@@ -601,6 +601,12 @@ public class TestRegistryDNS extends Assert {
assertEquals(4, registryDNS.getZoneCount());
}
+ @Test
+ public void testExampleDotCom() throws Exception {
+ Name name = Name.fromString("example.com.");
+ Record[] records = getRegistryDNS().getRecords(name, Type.SOA);
+ assertNotNull("example.com exists:", records);
+ }
public RegistryDNS getRegistryDNS() {
return registryDNS;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: Fixing Job History Server
Configuration parsing. (Jason Lowe via asuresh)
Posted by su...@apache.org.
Fixing Job History Server Configuration parsing. (Jason Lowe via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5eb7dbe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5eb7dbe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5eb7dbe9
Branch: refs/heads/YARN-5881
Commit: 5eb7dbe9b31a45f57f2e1623aa1c9ce84a56c4d1
Parents: dd07038
Author: Arun Suresh <as...@apache.org>
Authored: Thu Nov 9 15:15:51 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Nov 9 15:15:51 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 163 ++++++++++++++-----
.../mapreduce/v2/hs/HistoryFileManager.java | 2 +-
2 files changed, 122 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eb7dbe9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index f94eba6..dfbeec7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.conf;
+import com.ctc.wstx.api.ReaderConfig;
import com.ctc.wstx.io.StreamBootstrapper;
import com.ctc.wstx.io.SystemId;
import com.ctc.wstx.stax.WstxInputFactory;
@@ -70,6 +71,7 @@ import java.util.concurrent.atomic.AtomicReference;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@@ -91,6 +93,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -206,19 +209,31 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final String DEFAULT_STRING_CHECK =
"testingforemptydefaultvalue";
+ private static boolean restrictSystemPropsDefault = false;
+ private boolean restrictSystemProps = restrictSystemPropsDefault;
private boolean allowNullValueProperties = false;
private static class Resource {
private final Object resource;
private final String name;
+ private final boolean restrictParser;
public Resource(Object resource) {
this(resource, resource.toString());
}
-
+
+ public Resource(Object resource, boolean useRestrictedParser) {
+ this(resource, resource.toString(), useRestrictedParser);
+ }
+
public Resource(Object resource, String name) {
+ this(resource, name, getRestrictParserDefault(resource));
+ }
+
+ public Resource(Object resource, String name, boolean restrictParser) {
this.resource = resource;
this.name = name;
+ this.restrictParser = restrictParser;
}
public String getName(){
@@ -228,11 +243,28 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
public Object getResource() {
return resource;
}
-
+
+ public boolean isParserRestricted() {
+ return restrictParser;
+ }
+
@Override
public String toString() {
return name;
}
+
+ private static boolean getRestrictParserDefault(Object resource) {
+ if (resource instanceof String) {
+ return false;
+ }
+ UserGroupInformation user;
+ try {
+ user = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to determine current user", e);
+ }
+ return user.getRealUser() != null;
+ }
}
/**
@@ -254,7 +286,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
new ConcurrentHashMap<String, Boolean>());
private boolean loadDefaults = true;
-
+
/**
* Configuration objects
*/
@@ -777,6 +809,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.overlay = (Properties)other.overlay.clone();
}
+ this.restrictSystemProps = other.restrictSystemProps;
if (other.updatingResource != null) {
this.updatingResource = new ConcurrentHashMap<String, String[]>(
other.updatingResource);
@@ -825,6 +858,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
+ public static void setRestrictSystemPropertiesDefault(boolean val) {
+ restrictSystemPropsDefault = val;
+ }
+
+ public void setRestrictSystemProperties(boolean val) {
+ this.restrictSystemProps = val;
+ }
+
/**
* Add a configuration resource.
*
@@ -838,6 +879,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
addResourceObject(new Resource(name));
}
+ public void addResource(String name, boolean restrictedParser) {
+ addResourceObject(new Resource(name, restrictedParser));
+ }
+
/**
* Add a configuration resource.
*
@@ -852,6 +897,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
addResourceObject(new Resource(url));
}
+ public void addResource(URL url, boolean restrictedParser) {
+ addResourceObject(new Resource(url, restrictedParser));
+ }
+
/**
* Add a configuration resource.
*
@@ -866,6 +915,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
addResourceObject(new Resource(file));
}
+ public void addResource(Path file, boolean restrictedParser) {
+ addResourceObject(new Resource(file, restrictedParser));
+ }
+
/**
* Add a configuration resource.
*
@@ -883,6 +936,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
addResourceObject(new Resource(in));
}
+ public void addResource(InputStream in, boolean restrictedParser) {
+ addResourceObject(new Resource(in, restrictedParser));
+ }
+
/**
* Add a configuration resource.
*
@@ -896,7 +953,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
public void addResource(InputStream in, String name) {
addResourceObject(new Resource(in, name));
}
-
+
+ public void addResource(InputStream in, String name,
+ boolean restrictedParser) {
+ addResourceObject(new Resource(in, name, restrictedParser));
+ }
+
/**
* Add a configuration resource.
*
@@ -906,7 +968,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @param conf Configuration object from which to load properties
*/
public void addResource(Configuration conf) {
- addResourceObject(new Resource(conf.getProps()));
+ addResourceObject(new Resource(conf.getProps(), conf.restrictSystemProps));
}
@@ -926,6 +988,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private synchronized void addResourceObject(Resource resource) {
resources.add(resource); // add to resources
+ restrictSystemProps |= resource.isParserRestricted();
reloadConfiguration();
}
@@ -1034,34 +1097,36 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
final String var = eval.substring(varBounds[SUB_START_IDX],
varBounds[SUB_END_IDX]);
String val = null;
- try {
- if (var.startsWith("env.") && 4 < var.length()) {
- String v = var.substring(4);
- int i = 0;
- for (; i < v.length(); i++) {
- char c = v.charAt(i);
- if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
- val = getenv(v.substring(0, i));
- if (val == null || val.length() == 0) {
- val = v.substring(i + 2);
- }
- break;
- } else if (c == '-') {
- val = getenv(v.substring(0, i));
- if (val == null) {
- val = v.substring(i + 1);
+ if (!restrictSystemProps) {
+ try {
+ if (var.startsWith("env.") && 4 < var.length()) {
+ String v = var.substring(4);
+ int i = 0;
+ for (; i < v.length(); i++) {
+ char c = v.charAt(i);
+ if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
+ val = getenv(v.substring(0, i));
+ if (val == null || val.length() == 0) {
+ val = v.substring(i + 2);
+ }
+ break;
+ } else if (c == '-') {
+ val = getenv(v.substring(0, i));
+ if (val == null) {
+ val = v.substring(i + 1);
+ }
+ break;
}
- break;
}
+ if (i == v.length()) {
+ val = getenv(v);
+ }
+ } else {
+ val = getProperty(var);
}
- if (i == v.length()) {
- val = getenv(v);
- }
- } else {
- val = getProperty(var);
+ } catch (SecurityException se) {
+ LOG.warn("Unexpected SecurityException in Configuration", se);
}
- } catch(SecurityException se) {
- LOG.warn("Unexpected SecurityException in Configuration", se);
}
if (val == null) {
val = getRaw(var);
@@ -1128,6 +1193,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.allowNullValueProperties = val;
}
+ public void setRestrictSystemProps(boolean val) {
+ this.restrictSystemProps = val;
+ }
+
/**
* Return existence of the <code>name</code> property, but only for
* names which have no valid value, usually non-existent or commented
@@ -2718,7 +2787,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return configMap;
}
- private XMLStreamReader parse(URL url)
+ private XMLStreamReader parse(URL url, boolean restricted)
throws IOException, XMLStreamException {
if (!quietmode) {
if (LOG.isDebugEnabled()) {
@@ -2735,11 +2804,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
// with other users.
connection.setUseCaches(false);
}
- return parse(connection.getInputStream(), url.toString());
+ return parse(connection.getInputStream(), url.toString(), restricted);
}
- private XMLStreamReader parse(InputStream is, String systemIdStr)
- throws IOException, XMLStreamException {
+ private XMLStreamReader parse(InputStream is, String systemIdStr,
+ boolean restricted) throws IOException, XMLStreamException {
if (!quietmode) {
LOG.debug("parsing input stream " + is);
}
@@ -2747,9 +2816,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return null;
}
SystemId systemId = SystemId.construct(systemIdStr);
- return XML_INPUT_FACTORY.createSR(XML_INPUT_FACTORY.createPrivateConfig(),
- systemId, StreamBootstrapper.getInstance(null, systemId, is), false,
- true);
+ ReaderConfig readerConfig = XML_INPUT_FACTORY.createPrivateConfig();
+ if (restricted) {
+ readerConfig.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ }
+ return XML_INPUT_FACTORY.createSR(readerConfig, systemId,
+ StreamBootstrapper.getInstance(null, systemId, is), false, true);
}
private void loadResources(Properties properties,
@@ -2757,7 +2829,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
boolean quiet) {
if(loadDefaults) {
for (String resource : defaultResources) {
- loadResource(properties, new Resource(resource), quiet);
+ loadResource(properties, new Resource(resource, false), quiet);
}
}
@@ -2777,12 +2849,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
name = wrapper.getName();
XMLStreamReader2 reader = null;
boolean returnCachedProperties = false;
+ boolean isRestricted = wrapper.isParserRestricted();
if (resource instanceof URL) { // an URL resource
- reader = (XMLStreamReader2)parse((URL)resource);
+ reader = (XMLStreamReader2)parse((URL)resource, isRestricted);
} else if (resource instanceof String) { // a CLASSPATH resource
URL url = getResource((String)resource);
- reader = (XMLStreamReader2)parse(url);
+ reader = (XMLStreamReader2)parse(url, isRestricted);
} else if (resource instanceof Path) { // a file resource
// Can't use FileSystem API or we get an infinite loop
// since FileSystem uses Configuration API. Use java.io.File instead.
@@ -2793,10 +2866,12 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
LOG.debug("parsing File " + file);
}
reader = (XMLStreamReader2)parse(new BufferedInputStream(
- new FileInputStream(file)), ((Path)resource).toString());
+ new FileInputStream(file)), ((Path)resource).toString(),
+ isRestricted);
}
} else if (resource instanceof InputStream) {
- reader = (XMLStreamReader2)parse((InputStream)resource, null);
+ reader = (XMLStreamReader2)parse((InputStream)resource, null,
+ isRestricted);
returnCachedProperties = true;
} else if (resource instanceof Properties) {
overlay(properties, (Properties)resource);
@@ -2878,6 +2953,10 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (confInclude == null) {
break;
}
+ if (isRestricted) {
+ throw new RuntimeException("Error parsing resource " + wrapper
+ + ": XInclude is not supported for restricted resources");
+ }
// Determine if the included resource is a classpath resource
// otherwise fallback to a file resource
// xi:include are treated as inline and retain current source
@@ -2992,7 +3071,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (returnCachedProperties) {
overlay(properties, toAddTo);
- return new Resource(toAddTo, name);
+ return new Resource(toAddTo, name, wrapper.isParserRestricted());
}
return null;
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eb7dbe9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 61c750e..b418db7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -508,7 +508,7 @@ public class HistoryFileManager extends AbstractService {
public synchronized Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confFile), confFile.toString());
+ jobConf.addResource(fc.open(confFile), confFile.toString(), true);
return jobConf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-15012. Add readahead,
dropbehind, and unbuffer to StreamCapabilities. Contributed by John Zhuge.
Posted by su...@apache.org.
HADOOP-15012. Add readahead, dropbehind, and unbuffer to StreamCapabilities. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf6a6602
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf6a6602
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf6a6602
Branch: refs/heads/YARN-5881
Commit: bf6a660232b01642b07697a289c773ea5b97217c
Parents: 0a72c2f
Author: John Zhuge <jz...@apache.org>
Authored: Mon Nov 6 23:54:27 2017 -0800
Committer: John Zhuge <jz...@apache.org>
Committed: Thu Nov 9 10:16:12 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/FSDataInputStream.java | 15 +++---
.../apache/hadoop/fs/StreamCapabilities.java | 48 +++++++++++++-----
.../hadoop/fs/StreamCapabilitiesPolicy.java | 51 ++++++++++++++++++++
.../src/site/markdown/filesystem/filesystem.md | 21 +++++---
.../org/apache/hadoop/hdfs/DFSInputStream.java | 16 +++++-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 12 ++---
.../hadoop/fs/azure/BlockBlobAppendStream.java | 17 ++++---
7 files changed, 140 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
index a80279d..08d71f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.util.IdentityHashStore;
public class FSDataInputStream extends DataInputStream
implements Seekable, PositionedReadable,
ByteBufferReadable, HasFileDescriptor, CanSetDropBehind, CanSetReadahead,
- HasEnhancedByteBufferAccess, CanUnbuffer {
+ HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities {
/**
* Map ByteBuffers that we have handed out to readers to ByteBufferPool
* objects
@@ -227,12 +227,15 @@ public class FSDataInputStream extends DataInputStream
@Override
public void unbuffer() {
- try {
- ((CanUnbuffer)in).unbuffer();
- } catch (ClassCastException e) {
- throw new UnsupportedOperationException("this stream " +
- in.getClass().getName() + " does not " + "support unbuffering.");
+ StreamCapabilitiesPolicy.unbuffer(in);
+ }
+
+ @Override
+ public boolean hasCapability(String capability) {
+ if (in instanceof StreamCapabilities) {
+ return ((StreamCapabilities) in).hasCapability(capability);
}
+ return false;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
index 65aa679..3549cdc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilities.java
@@ -23,27 +23,49 @@ import org.apache.hadoop.classification.InterfaceStability;
/**
* Interface to query streams for supported capabilities.
+ *
+ * Capability strings must be in lower case.
+ *
+ * Constant strings are chosen over enums in order to allow other file systems
+ * to define their own capabilities.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public interface StreamCapabilities {
/**
+ * Stream hflush capability implemented by {@link Syncable#hflush()}.
+ */
+ String HFLUSH = "hflush";
+
+ /**
+ * Stream hsync capability implemented by {@link Syncable#hsync()}.
+ */
+ String HSYNC = "hsync";
+
+ /**
+ * Stream setReadahead capability implemented by
+ * {@link CanSetReadahead#setReadahead(Long)}.
+ */
+ String READAHEAD = "in:readahead";
+
+ /**
+ * Stream setDropBehind capability implemented by
+ * {@link CanSetDropBehind#setDropBehind(Boolean)}.
+ */
+ String DROPBEHIND = "dropbehind";
+
+ /**
+ * Stream unbuffer capability implemented by {@link CanUnbuffer#unbuffer()}.
+ */
+ String UNBUFFER = "in:unbuffer";
+
+ /**
* Capabilities that a stream can support and be queried for.
*/
+ @Deprecated
enum StreamCapability {
- /**
- * Stream hflush capability to flush out the data in client's buffer.
- * Streams with this capability implement {@link Syncable} and support
- * {@link Syncable#hflush()}.
- */
- HFLUSH("hflush"),
-
- /**
- * Stream hsync capability to flush out the data in client's buffer and
- * the disk device. Streams with this capability implement {@link Syncable}
- * and support {@link Syncable#hsync()}.
- */
- HSYNC("hsync");
+ HFLUSH(StreamCapabilities.HFLUSH),
+ HSYNC(StreamCapabilities.HSYNC);
private final String capability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
new file mode 100644
index 0000000..3080780
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.InputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Static methods to implement policies for {@link StreamCapabilities}.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class StreamCapabilitiesPolicy {
+ /**
+ * Implement the policy for {@link CanUnbuffer#unbuffer()}.
+ *
+ * @param in the input stream
+ */
+ public static void unbuffer(InputStream in) {
+ try {
+ if (in instanceof StreamCapabilities
+ && ((StreamCapabilities) in).hasCapability(
+ StreamCapabilities.UNBUFFER)) {
+ ((CanUnbuffer) in).unbuffer();
+ }
+ } catch (ClassCastException e) {
+ throw new UnsupportedOperationException("this stream " +
+ in.getClass().getName() +
+ " claims to unbuffer but forgets to implement CanUnbuffer");
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index d251804..c0e2a2c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -1359,7 +1359,8 @@ problems were not considered during the implementation of these loops.
## <a name="StreamCapability"></a> interface `StreamCapabilities`
The `StreamCapabilities` provides a way to programmatically query the
-capabilities that an `OutputStream` supports.
+capabilities that `OutputStream`, `InputStream`, or other FileSystem class
+supports.
```java
public interface StreamCapabilities {
@@ -1369,12 +1370,16 @@ public interface StreamCapabilities {
### `boolean hasCapability(capability)`
-Return true if the `OutputStream` has the desired capability.
+Return true if the `OutputStream`, `InputStream`, or other FileSystem class
+has the desired capability.
The caller can query the capabilities of a stream using a string value.
-It currently supports to query:
-
- * `StreamCapabilties.HFLUSH` ("*hflush*"): the capability to flush out the data
- in client's buffer.
- * `StreamCapabilities.HSYNC` ("*hsync*"): capability to flush out the data in
- client's buffer and the disk device.
+Here is a table of possible string values:
+
+String | Constant | Implements | Description
+-------------|------------|------------------|-------------------------------
+hflush | HFLUSH | Syncable | Flush out the data in client's user buffer. After the return of this call, new readers will see the data.
+hsync | HSYNC | Syncable | Flush out the data in client's user buffer all the way to the disk device (but the disk may have it in its cache). Similar to POSIX fsync.
+in:readahead | READAHEAD | CanSetReadahead | Set the readahead on the input stream.
+dropbehind | DROPBEHIND | CanSetDropBehind | Drop the cache.
+in:unbuffer | UNBUFFER | CanUnbuffer | Reduce the buffering on the input stream.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index e3d7ade..d3d6669 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
import org.apache.hadoop.hdfs.client.impl.BlockReaderFactory;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
@@ -81,6 +82,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.IdentityHashStore;
import org.apache.hadoop.util.StopWatch;
+import org.apache.hadoop.util.StringUtils;
import org.apache.htrace.core.SpanId;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
@@ -96,7 +98,7 @@ import javax.annotation.Nonnull;
@InterfaceAudience.Private
public class DFSInputStream extends FSInputStream
implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
- HasEnhancedByteBufferAccess, CanUnbuffer {
+ HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities {
@VisibleForTesting
public static boolean tcpReadsDisabledForTesting = false;
private long hedgedReadOpsLoopNumForTesting = 0;
@@ -1779,4 +1781,16 @@ public class DFSInputStream extends FSInputStream
public synchronized void unbuffer() {
closeCurrentBlockReaders();
}
+
+ @Override
+ public boolean hasCapability(String capability) {
+ switch (StringUtils.toLowerCase(capability)) {
+ case StreamCapabilities.READAHEAD:
+ case StreamCapabilities.DROPBEHIND:
+ case StreamCapabilities.UNBUFFER:
+ return true;
+ default:
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 83f1425..7849796 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs;
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HFLUSH;
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HSYNC;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -69,6 +66,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.DataChecksum.Type;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.htrace.core.TraceScope;
import org.slf4j.Logger;
@@ -552,11 +550,13 @@ public class DFSOutputStream extends FSOutputSummer
@Override
public boolean hasCapability(String capability) {
- if (capability.equalsIgnoreCase(HSYNC.getValue()) ||
- capability.equalsIgnoreCase((HFLUSH.getValue()))) {
+ switch (StringUtils.toLowerCase(capability)) {
+ case StreamCapabilities.HSYNC:
+ case StreamCapabilities.HFLUSH:
return true;
+ default:
+ return false;
}
- return false;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf6a6602/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index 84342cd..9a85308 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.UUID;
import java.util.Random;
import java.util.concurrent.ConcurrentLinkedDeque;
@@ -63,9 +64,6 @@ import com.microsoft.azure.storage.blob.BlockEntry;
import com.microsoft.azure.storage.blob.BlockListingFilter;
import com.microsoft.azure.storage.blob.BlockSearchMode;
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HFLUSH;
-import static org.apache.hadoop.fs.StreamCapabilities.StreamCapability.HSYNC;
-
/**
* Stream object that implements append for Block Blobs in WASB.
*
@@ -550,9 +548,16 @@ public class BlockBlobAppendStream extends OutputStream implements Syncable,
*/
@Override
public boolean hasCapability(String capability) {
- return compactionEnabled
- && (capability.equalsIgnoreCase(HSYNC.getValue())
- || capability.equalsIgnoreCase((HFLUSH.getValue())));
+ if (!compactionEnabled) {
+ return false;
+ }
+ switch (capability.toLowerCase(Locale.ENGLISH)) {
+ case StreamCapabilities.HSYNC:
+ case StreamCapabilities.HFLUSH:
+ return true;
+ default:
+ return false;
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage
issue in RegistryDNS. Contributed by Jian He
Posted by su...@apache.org.
YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc
Branch: refs/heads/YARN-5881
Commit: d2775fbccaa187729d86459542210434ce4c3b8b
Parents: 501be9b
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Oct 25 13:01:01 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/registry/server/dns/RegistryDNS.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 37e8429..9097c46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
addNIOUDP(addr, port);
- addNIOTCP(addr, port);
+ //TODO Fix dns lookup over TCP
+// addNIOTCP(addr, port);
}
/**
@@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
byte[] response = null;
try {
query = new Message(in);
+ LOG.info("received TCP query {}", query.getQuestion());
response = generateReply(query, ch.socket());
if (response == null) {
return;
@@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
input.flip();
input.get(in);
query = new Message(in);
- LOG.info("{}: received query {}", remoteAddress,
+ LOG.info("{}: received UDP query {}", remoteAddress,
query.getQuestion());
response = generateReply(query, null);
if (response == null) {
@@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
output.put(response);
output.flip();
- LOG.info("{}: sending response", remoteAddress);
+ LOG.debug("{}: sending response", remoteAddress);
channel.send(output, remoteAddress);
}
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-7143. FileNotFound handling
in ResourceUtils is inconsistent
Posted by su...@apache.org.
YARN-7143. FileNotFound handling in ResourceUtils is inconsistent
Change-Id: Ib1bb487e14a15edd2b5a42cf5078c5a2b295f069
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/462f6c49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/462f6c49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/462f6c49
Branch: refs/heads/YARN-5881
Commit: 462f6c490efd2a38a9ba639bcda47b3aa667f650
Parents: a1382a1
Author: Daniel Templeton <te...@apache.org>
Authored: Thu Nov 9 10:36:49 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Thu Nov 9 11:58:49 2017 -0800
----------------------------------------------------------------------
.../yarn/util/resource/ResourceUtils.java | 56 ++++++++------------
1 file changed, 23 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462f6c49/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index c9cc27b..9c9c0ef8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -342,17 +342,14 @@ public class ResourceUtils {
if (!initializedResources) {
synchronized (ResourceUtils.class) {
if (!initializedResources) {
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- try {
- addResourcesFileToConf(resourceFile, conf);
- } catch (FileNotFoundException fe) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unable to find '" + resourceFile + "'.");
- }
+ Configuration resConf = conf;
+
+ if (resConf == null) {
+ resConf = new YarnConfiguration();
}
- initializeResourcesMap(conf);
+
+ addResourcesFileToConf(resourceFile, resConf);
+ initializeResourcesMap(resConf);
}
}
}
@@ -389,7 +386,7 @@ public class ResourceUtils {
}
private static void addResourcesFileToConf(String resourceFile,
- Configuration conf) throws FileNotFoundException {
+ Configuration conf) {
try {
InputStream ris = getConfInputStream(resourceFile, conf);
if (LOG.isDebugEnabled()) {
@@ -397,15 +394,11 @@ public class ResourceUtils {
}
conf.addResource(ris);
} catch (FileNotFoundException fe) {
- throw fe;
- } catch (IOException ie) {
+ LOG.info("Unable to find '" + resourceFile + "'.");
+ } catch (IOException | YarnException ex) {
LOG.fatal("Exception trying to read resource types configuration '"
- + resourceFile + "'.", ie);
- throw new YarnRuntimeException(ie);
- } catch (YarnException ye) {
- LOG.fatal("YARN Exception trying to read resource types configuration '"
- + resourceFile + "'.", ye);
- throw new YarnRuntimeException(ye);
+ + resourceFile + "'.", ex);
+ throw new YarnRuntimeException(ex);
}
}
@@ -467,22 +460,19 @@ public class ResourceUtils {
private static Map<String, ResourceInformation> initializeNodeResourceInformation(
Configuration conf) {
Map<String, ResourceInformation> nodeResources = new HashMap<>();
- try {
- addResourcesFileToConf(
- YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
- for (Map.Entry<String, String> entry : conf) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
- }
- } catch (FileNotFoundException fe) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Couldn't find node resources file: "
- + YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE);
+
+ addResourcesFileToConf(YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE,
+ conf);
+
+ for (Map.Entry<String, String> entry : conf) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+ addResourceInformation(key, value, nodeResources);
}
}
+
return nodeResources;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
deleted file mode 100644
index 65f0472..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- *
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_RESOLVE
- + " "
- + ARG_PATH + " <path> "
- + "[" + ARG_LIST + "] "
- + "[" + ARG_OUTPUT + " <filename> ] "
- + "[" + ARG_DESTDIR + " <directory> ] "
- ;
- public ActionResolveArgs() {
- }
-
- @Override
- public String getActionName() {
- return ACTION_RESOLVE;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_PATH},
- description = "resolve a path")
- public String path;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "destination directory for operations")
- public File destdir;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "dest file")
- public File out;
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_RESOLVE).append(" ");
- sb.append(ARG_PATH).append(" ").append(path).append(" ");
- if (list) {
- sb.append(ARG_LIST).append(" ");
- }
- if (destdir != null) {
- sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
- }
- if (out != null) {
- sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
- }
- return sb.toString();
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (StringUtils.isEmpty(path)) {
- throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
- }
- if (list && out != null) {
- throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
- }
- if (out != null && destdir != null) {
- throw new BadCommandArgumentsException(
- ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
- );
- }
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public boolean isList() {
- return list;
- }
-
- public void setList(boolean list) {
- this.list = list;
- }
-
- public File getDestdir() {
- return destdir;
- }
-
- public void setDestdir(File destdir) {
- this.destdir = destdir;
- }
-
- public File getOut() {
- return out;
- }
-
- public void setOut(File out) {
- this.out = out;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
deleted file mode 100644
index b03dc92..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
- commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
-
-public class ActionResourceArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_RESOURCE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install the resource(s)")
- public boolean install;
-
- @Parameter(names = {ARG_DELETE},
- description = "Delete the file")
- public boolean delete;
-
- @Parameter(names = {ARG_LIST},
- description = "List of installed files")
- public boolean list;
-
- @Parameter(names = {ARG_RESOURCE},
- description = "Name of the file or directory")
- public String resource;
-
- @Parameter(names = {ARG_DESTDIR},
- description = "The name of the folder in which to store the resources")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
deleted file mode 100644
index 31f25ef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_STATUS},
- commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
-
-public class ActionStatusArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STATUS;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the status information")
- public String output;
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Lifetime of the service from the time of request")
- public boolean lifetime;
-
- public String getOutput() {
- return output;
- }
-
- public void setOutput(String output) {
- this.output = output;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
deleted file mode 100644
index 175e367..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_START },
- commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
-public class ActionThawArgs extends AbstractActionArgs implements
- WaitTimeAccessor,
- LaunchArgsAccessor {
-
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_START;
- }
-
- @Override
- public int getWaittime() {
- return launchArgs.getWaittime();
- }
-
- @ParametersDelegate
- LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
-
- @Parameter(names = {ARG_LIFETIME},
- description = "Life time of the service since service started at"
- + " running state")
- public long lifetime;
-
- @Override
- public String getRmAddress() {
- return launchArgs.getRmAddress();
- }
-
- @Override
- public void setWaittime(int waittime) {
- launchArgs.setWaittime(waittime);
- }
-
-
- @Override
- public File getOutputFile() {
- return launchArgs.getOutputFile();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
deleted file mode 100644
index cf48513..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
- commandDescription = "save tokens to a file or list tokens in a file")
-public class ActionTokensArgs extends AbstractActionArgs {
-
- public static final String DUPLICATE_ARGS = "Only one of " +
- ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
-
- public static final String MISSING_KT_PROVIDER =
- "Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
- + " must be provided";
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_TOKENS;
- }
-
- @Parameter(names = {ARG_OUTPUT},
- description = "File to write")
- public File output;
-
- @Parameter(names = {ARG_SOURCE},
- description = "source file")
- public File source;
-
- @Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
- public File keytab;
-
- @Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
- public String principal="";
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
- if (output != null && source != null) {
- throw new BadCommandArgumentsException(DUPLICATE_ARGS);
- }
-
- // this is actually a !xor
- if (keytab != null ^ !principal.isEmpty()) {
- throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
deleted file mode 100644
index 00af69a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
-
-public class ActionUpdateArgs extends AbstractActionArgs {
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
- public long lifetime;
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_UPDATE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
deleted file mode 100644
index 00151f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Static argument manipulation operations
- */
-public class ArgOps {
-
- private static final Logger
- log = LoggerFactory.getLogger(ArgOps.class);
-
- /**
- * create a 3-tuple
- */
- public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<>(3);
- l.add(msg);
- l.add(min);
- l.add(max);
- return l;
- }
-
- public static void applyFileSystemBinding(String filesystemBinding,
- Configuration conf) {
- if (filesystemBinding != null) {
- //filesystem argument was set -this overwrites any defaults in the
- //configuration
- FileSystem.setDefaultUri(conf, filesystemBinding);
- }
- }
-
- public static void splitPairs(Collection<String> pairs,
- Map<String, String> dest) {
- for (String prop : pairs) {
- String[] keyval = prop.split("=", 2);
- if (keyval.length == 2) {
- dest.put(keyval[0], keyval[1]);
- }
- }
- }
-
-
- public static void applyDefinitions(Map<String, String> definitionMap,
- Configuration conf) {
- for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- log.debug("configuration[{}]<=\"{}\"", key, val);
- conf.set(key, val, "command line");
- }
- }
-
- /**
- * Create a map from a tuple list like ['worker','2','master','1] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
- * @param description description for errors
- * @param list list to conver to tuples
- * @return the map of key value pairs -unordered.
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, String> convertTupleListToMap(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, String> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 2 != 0) {
- //odd number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 2) {
- String key = list.get(count);
- String val = list.get(count + 1);
- if (results.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": " + key);
- }
- results.put(key, val);
- }
- }
- return results;
- }
-
- /**
- * Create a map from a tuple list like
- * ['worker','heapsize','5G','master','heapsize','2M'] into a map
- * ['worker':'2',"master":'1'];
- * Duplicate entries also trigger errors
-
- * @throws BadCommandArgumentsException odd #of arguments received
- */
- public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws BadCommandArgumentsException {
-
- Map<String, Map<String, String>> results = new HashMap<>();
- if (list != null && !list.isEmpty()) {
- int size = list.size();
- if (size % 3 != 0) {
- //wrong number of elements, not permitted
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_PARSE_FAILURE + description);
- }
- for (int count = 0; count < size; count += 3) {
- String role = list.get(count);
- String key = list.get(count + 1);
- String val = list.get(count + 2);
- Map<String, String> roleMap = results.get(role);
- if (roleMap == null) {
- //demand create new role map
- roleMap = new HashMap<>();
- results.put(role, roleMap);
- }
- if (roleMap.get(key) != null) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_DUPLICATE_ENTRY + description
- + ": for key " + key + " under " + role);
- }
- roleMap.put(key, val);
- }
- }
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
deleted file mode 100644
index 67571e2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Here are all the arguments that may be parsed by the client or server
- * command lines.
- *
- * Important: Please keep the main list in alphabetical order
- * so it is easier to see what arguments are there
- */
-public interface Arguments {
-
- String ARG_FILE = "--file";
- String ARG_FILE_SHORT = "-f";
- String ARG_BASE_PATH = "--basepath";
- String ARG_COMPONENT = "--component";
- String ARG_COMPONENT_SHORT = "--comp";
- String ARG_COMPONENTS = "--components";
- String ARG_COMP_OPT= "--compopt";
- String ARG_COMP_OPT_SHORT = "--co";
- String ARG_CONFIG = "--config";
- String ARG_CONTAINERS = "--containers";
- String ARG_DEBUG = "--debug";
- String ARG_DEFINE = "-D";
- String ARG_DELETE = "--delete";
- String ARG_DEST = "--dest";
- String ARG_DESTDIR = "--destdir";
- String ARG_EXAMPLE = "--example";
- String ARG_EXAMPLE_SHORT = "-e";
- String ARG_FOLDER = "--folder";
- String ARG_FORCE = "--force";
- String ARG_FORMAT = "--format";
- String ARG_GETCONF = "--getconf";
- String ARG_GETEXP = "--getexp";
- String ARG_GETFILES = "--getfiles";
- String ARG_HELP = "--help";
- String ARG_IMAGE = "--image";
- String ARG_INSTALL = "--install";
- String ARG_INTERNAL = "--internal";
- String ARG_KEYLEN = "--keylen";
- String ARG_KEYTAB = "--keytab";
- String ARG_KEYTABINSTALL = ARG_INSTALL;
- String ARG_KEYTABDELETE = ARG_DELETE;
- String ARG_KEYTABLIST = "--list";
- String ARG_LIST = "--list";
- String ARG_LISTCONF = "--listconf";
- String ARG_LISTEXP = "--listexp";
- String ARG_LISTFILES = "--listfiles";
- String ARG_LIVE = "--live";
- String ARG_MANAGER = "--manager";
- String ARG_MANAGER_SHORT = "--m";
- String ARG_MESSAGE = "--message";
- String ARG_NAME = "--name";
- String ARG_OPTION = "--option";
- String ARG_OPTION_SHORT = "-O";
- String ARG_OUTPUT = "--out";
- String ARG_OUTPUT_SHORT = "-o";
- String ARG_OVERWRITE = "--overwrite";
- String ARG_PACKAGE = "--package";
- String ARG_PATH = "--path";
- String ARG_PRINCIPAL = "--principal";
- String ARG_QUEUE = "--queue";
- String ARG_SHORT_QUEUE = "-q";
- String ARG_LIFETIME = "--lifetime";
- String ARG_RESOURCE = "--resource";
- String ARG_RESOURCE_MANAGER = "--rm";
- String ARG_SECURE = "--secure";
- String ARG_SERVICETYPE = "--servicetype";
- String ARG_SERVICES = "--services";
- String ARG_SOURCE = "--source";
- String ARG_STATE = "--state";
- String ARG_SYSPROP = "-S";
- String ARG_USER = "--user";
- String ARG_UPLOAD = "--upload";
- String ARG_VERBOSE = "--verbose";
- String ARG_VERSION = "--version";
- String ARG_WAIT = "--wait";
-/*
- STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
- RIGHT PLACE IN THE LIST
- */
-
- // Tha path in hdfs to be read by Service AM
- String ARG_SERVICE_DEF_PATH = "-cluster-uri";
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
deleted file mode 100644
index 84d3bbd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-
-import java.util.Collection;
-
-/**
- * Client CLI Args
- */
-
-public class ClientArgs extends CommonArgs {
-
- // =========================================================
- // Keep all of these in alphabetical order. Thanks.
- // =========================================================
-
- private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionClientArgs actionClientArgs = new ActionClientArgs();
- private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
- private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
- private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
- private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
- private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
- private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
- private final ActionListArgs actionListArgs = new ActionListArgs();
- private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
- private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
- private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
- private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
- private final ActionThawArgs actionThawArgs = new ActionThawArgs();
- private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
-
- public ClientArgs(String[] args) {
- super(args);
- }
-
- public ClientArgs(Collection args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
-
- addActions(
- actionBuildArgs,
- actionCreateArgs,
- actionDependencyArgs,
- actionDestroyArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionHelpArgs,
- actionStatusArgs,
- actionThawArgs
- );
- }
-
- @Override
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- super.applyDefinitions(conf);
- }
-
-
- public ActionBuildArgs getActionBuildArgs() {
- return actionBuildArgs;
- }
-
- public ActionUpdateArgs getActionUpdateArgs() {
- return actionUpdateArgs;
- }
-
- public ActionCreateArgs getActionCreateArgs() {
- return actionCreateArgs;
- }
-
- public ActionDependencyArgs getActionDependencyArgs() {
- return actionDependencyArgs;
- }
-
- public ActionFlexArgs getActionFlexArgs() {
- return actionFlexArgs;
- }
-
- /**
- * Look at the chosen action and bind it as the core action for the operation.
- * @throws SliderException bad argument or similar
- */
- @Override
- public void applyAction() throws SliderException {
- String action = getAction();
- if (SliderUtils.isUnset(action)) {
- action = ACTION_HELP;
- }
- switch (action) {
- case ACTION_BUILD:
- bindCoreAction(actionBuildArgs);
- break;
-
- case ACTION_CREATE:
- bindCoreAction(actionCreateArgs);
- break;
-
- case ACTION_STOP:
- bindCoreAction(actionFreezeArgs);
- break;
-
- case ACTION_START:
- bindCoreAction(actionThawArgs);
- break;
-
- case ACTION_DEPENDENCY:
- bindCoreAction(actionDependencyArgs);
- break;
-
- case ACTION_DESTROY:
- bindCoreAction(actionDestroyArgs);
- break;
-
- case ACTION_EXISTS:
- bindCoreAction(actionExistsArgs);
- break;
-
- case ACTION_FLEX:
- bindCoreAction(actionFlexArgs);
- break;
-
- case ACTION_HELP:
- bindCoreAction(actionHelpArgs);
- break;
-
- case ACTION_KEYTAB:
- bindCoreAction(actionKeytabArgs);
- break;
-
- case ACTION_LIST:
- bindCoreAction(actionListArgs);
- break;
-
- case ACTION_REGISTRY:
- bindCoreAction(actionRegistryArgs);
- break;
-
- case ACTION_RESOLVE:
- bindCoreAction(actionResolveArgs);
- break;
-
- case ACTION_RESOURCE:
- bindCoreAction(actionResourceArgs);
- break;
-
- case ACTION_STATUS:
- bindCoreAction(actionStatusArgs);
- break;
-
- case ACTION_TOKENS:
- bindCoreAction(actionTokenArgs);
- break;
-
- case ACTION_UPDATE:
- bindCoreAction(actionUpdateArgs);
- break;
- default:
- throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
- + " " + action);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
deleted file mode 100644
index 145b44a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterDescription;
-import com.beust.jcommander.ParameterException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This class contains the common argument set for all tne entry points,
- * and the core parsing logic to verify that the action is on the list
- * of allowed actions -and that the remaining number of arguments is
- * in the range allowed
- */
-
-public abstract class CommonArgs extends ArgOps implements SliderActions,
- Arguments {
-
- protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
-
-
- private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
-
-
- @Parameter(names = ARG_HELP, help = true)
- public boolean help;
-
-
- /**
- -D name=value
-
- Define an HBase configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- public Map<String, String> definitionMap = new HashMap<String, String>();
- /**
- * System properties
- */
- public Map<String, String> syspropsMap = new HashMap<String, String>();
-
-
- /**
- * fields
- */
- public final JCommander commander;
- private final String[] args;
-
- private AbstractActionArgs coreAction;
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getClusterName() {
- return coreAction.getServiceName();
- }
-
- protected CommonArgs(String[] args) {
- this.args = args;
- commander = new JCommander(this);
- }
-
- protected CommonArgs(Collection args) {
- List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
- this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
- commander = new JCommander(this);
- }
-
- public String usage() {
- return usage(this, null);
- }
-
- public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
- String result = null;
- StringBuilder helperMessage = new StringBuilder();
- if (commandOfInterest == null) {
- // JCommander.usage is too verbose for a command with many options like
- // slider no short version of that is found Instead, we compose our msg by
- helperMessage.append("\nUsage: service COMMAND [options]\n");
- helperMessage.append("where COMMAND is one of\n");
- for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
- serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
- }
- helperMessage
- .append("Most commands print help when invoked without parameters or with --help");
- result = helperMessage.toString();
- } else {
- helperMessage.append("\nUsage: service ").append(commandOfInterest);
- helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <service>" : "");
- helperMessage.append("\n");
- for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
- .get(commandOfInterest).getParameters()) {
- String optional = paramDesc.getParameter().required() ? " (required)"
- : " (optional)";
- String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
- .getLongestName() : paramDesc.getLongestName() + " <"
- + paramDesc.getParameterized().getName() + ">";
- helperMessage.append(String.format("\t%-"
- + DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
- paramDesc.getDescription() + optional + "\n"));
- result = helperMessage.toString();
- }
- }
- return result;
- }
-
- /**
- * Parse routine -includes registering the action-specific argument classes
- * and postprocess it
- * @throws SliderException on any problem
- */
- public void parse() throws SliderException {
- addActionArguments();
- try {
- commander.parse(args);
- } catch (ParameterException e) {
- commander.usage(args[0]);
- throw e;
- }
- //now copy back to this class some of the attributes that are common to all
- //actions
- postProcess();
- }
-
-
- protected void addActions(Object... actions) {
- for (Object action : actions) {
- commander.addCommand(action);
- }
- }
-
- /**
- * Override point to add a set of actions
- */
- protected void addActionArguments() {
-
- }
-
- /**
- * validate args via {@link #validate()}
- * then postprocess the arguments
- */
- public void postProcess() throws SliderException {
- applyAction();
- validate();
-
- //apply entry set
- for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
- System.setProperty(entry.getKey(), entry.getValue());
- }
- }
-
-
- /**
- * Implementors must implement their action apply routine here
- */
- public abstract void applyAction() throws SliderException;
-
-
- /**
- * Bind the core action; this extracts any attributes that are used
- * across routines
- * @param action action to bind
- */
- protected void bindCoreAction(AbstractActionArgs action) {
- coreAction = action;
-
- splitPairs(coreAction.definitions, definitionMap);
- splitPairs(coreAction.sysprops, syspropsMap);
- }
-
- /**
- * Validate the arguments against the action requested
- */
- public void validate() throws BadCommandArgumentsException, UsageException {
- if (coreAction == null) {
- throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
- }
- log.debug("action={}", getAction());
- // let the action validate itself
- try {
- coreAction.validate();
- } catch (BadCommandArgumentsException e) {
- String badArgMsgBuilder =
- e.getMessage() + System.lineSeparator() + usage(this,
- coreAction.getActionName());
- throw new BadCommandArgumentsException(badArgMsgBuilder);
- }
- }
-
- /**
- * Apply all the definitions on the command line to the configuration
- * @param conf config
- */
- public void applyDefinitions(Configuration conf) throws
- BadCommandArgumentsException {
- applyDefinitions(definitionMap, conf);
- }
-
- public boolean isDebug() {
- return coreAction.debug;
- }
-
-
- public String getAction() {
- return commander.getParsedCommand();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
deleted file mode 100644
index b6cd0a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class ComponentArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * This is a listing of the roles to create
- */
- @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
- arity = 2,
- description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
- splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<>(0);
-
-
- /**
- * Get the role mapping (may be empty, but never null)
- * @return role mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
- return convertTupleListToMap("component", componentTuples);
- }
-
- public List<String> getComponentTuples() {
- return componentTuples;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
deleted file mode 100644
index 85de615..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.IParameterSplitter;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DontSplitArguments implements IParameterSplitter {
-
- @Override
- public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
- list.add(value);
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
deleted file mode 100644
index bf194b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.io.File;
-
-/**
- * Launch args for create and start and anything else that can start something
- */
-public interface LaunchArgsAccessor extends WaitTimeAccessor {
- String getRmAddress();
-
- File getOutputFile();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
deleted file mode 100644
index d42510c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Any launch-time args
- */
-public class LaunchArgsDelegate extends WaitArgsDelegate implements
- LaunchArgsAccessor {
-
-
- //TODO: do we need this?
- @Parameter(names = ARG_RESOURCE_MANAGER,
- description = "Resource manager hostname:port ",
- required = false)
- private String rmAddress;
-
- @Override
- public String getRmAddress() {
- return rmAddress;
- }
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any service report")
- public File outputFile;
-
- @Override
- public File getOutputFile() {
- return outputFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
deleted file mode 100644
index 7972716..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options.
- */
-public class OptionArgsDelegate extends AbstractArgsDelegate {
-
- /**
- * Options key value.
- */
- @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
- description = ARG_OPTION + "<name> <value>",
- splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
-
-
- /**
- * All the app component option triples.
- */
- @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
- description = "Component option " + ARG_COMP_OPT +
- " <component> <name> <option>",
- splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
-
- public Map<String, String> getOptionsMap() throws
- BadCommandArgumentsException {
- return convertTupleListToMap(ARG_OPTION, optionTuples);
- }
-
- /**
- * Get the role heap mapping (may be empty, but never null).
- * @return role heap mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, Map<String, String>> getCompOptionMap()
- throws BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
deleted file mode 100644
index 040ac64..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.converters.BaseConverter;
-import org.apache.hadoop.fs.Path;
-
-public class PathArgumentConverter extends BaseConverter<Path> {
-
- public PathArgumentConverter(String optionName) {
- super(optionName);
- }
-
- @Override
- public Path convert(String value) {
- return new Path(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
deleted file mode 100644
index 800bb24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Parameters sent by the Client to the AM
- */
-public class ServiceAMArgs extends CommonArgs {
-
- ServiceAMCreateAction createAction = new ServiceAMCreateAction();
-
- public ServiceAMArgs(String[] args) {
- super(args);
- }
-
- @Override
- protected void addActionArguments() {
- addActions(createAction);
- }
-
- // This is the path in hdfs to the service definition JSON file
- public String getServiceDefPath() {
- return createAction.serviceDefPath;
- }
-
- /**
- * Am binding is simple: there is only one action
- */
- @Override
- public void applyAction() {
- bindCoreAction(createAction);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
deleted file mode 100644
index 384e8ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ServiceAMCreateAction extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Parameter(names = ARG_SERVICE_DEF_PATH,
- description = "Path to the service definition JSON file", required = true)
- public String serviceDefPath;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
deleted file mode 100644
index 680bc0f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-/**
- * Actions by client
- */
-public interface SliderActions {
- String ACTION_BUILD = "build";
- String ACTION_CLIENT = "client";
- String ACTION_CREATE = "create";
- String ACTION_DEPENDENCY = "dependency";
- String ACTION_UPDATE = "update";
- String ACTION_UPGRADE = "upgrade";
- String ACTION_DESTROY = "destroy";
- String ACTION_EXISTS = "exists";
- String ACTION_FLEX = "flex";
- String ACTION_STOP = "stop";
- String ACTION_HELP = "help";
- String ACTION_INSTALL_KEYTAB = "install-keytab";
- String ACTION_KEYTAB = "keytab";
- String ACTION_LIST = "list";
-
- String ACTION_REGISTRY = "registry";
- String ACTION_RESOLVE = "resolve";
- String ACTION_RESOURCE = "resource";
- String ACTION_STATUS = "status";
- String ACTION_START = "start";
- String ACTION_TOKENS = "tokens";
-
- String DESCRIBE_ACTION_BUILD =
- "Build a service specification, but do not start it";
- String DESCRIBE_ACTION_CREATE =
- "Create a service, it's equivalent to first invoke build and then start";
- String DESCRIBE_ACTION_DEPENDENCY =
- "Yarn service framework dependency (libraries) management";
- String DESCRIBE_ACTION_UPDATE =
- "Update template for service";
- String DESCRIBE_ACTION_UPGRADE =
- "Rolling upgrade/downgrade the component/containerto a newer/previous version";
- String DESCRIBE_ACTION_DESTROY =
- "Destroy a stopped service, service must be stopped first before destroying.";
- String DESCRIBE_ACTION_EXISTS =
- "Probe for a service running";
- String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
- String DESCRIBE_ACTION_FREEZE =
- "Stop a running service";
- String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
- String DESCRIBE_ACTION_HELP = "Print help information";
- String DESCRIBE_ACTION_LIST =
- "List running services";
- String DESCRIBE_ACTION_REGISTRY =
- "Query the registry of a service";
- String DESCRIBE_ACTION_STATUS =
- "Get the status of a service";
- String DESCRIBE_ACTION_THAW =
- "Start a service with pre-built specification or a previously stopped service";
- String DESCRIBE_ACTION_CLIENT = "Install the service client in the specified directory or obtain a client keystore or truststore";
- String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
- String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
-
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
deleted file mode 100644
index 86f3709..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-public class WaitArgsDelegate extends AbstractArgsDelegate implements
- WaitTimeAccessor {
-
-
- //--wait [timeout]
- @Parameter(names = {ARG_WAIT},
- description = "time to wait for an action to complete")
- public int waittime = 0;
-
-
- @Override
- public int getWaittime() {
- return waittime;
- }
-
- @Override
- public void setWaittime(int waittime) {
- this.waittime = waittime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
deleted file mode 100644
index f6afae6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-public interface WaitTimeAccessor {
- int getWaittime();
- void setWaittime(int waittime);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 7e53d18..2809dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
@@ -484,8 +482,8 @@ public final class SliderUtils {
classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
} else {
log.info(
- "For faster submission of apps, upload dependencies using cmd {} {}",
- SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ "For faster submission of apps, upload dependencies using cmd " +
+ "enableFastLaunch");
}
classpath.addRemoteClasspathEnvVar();
classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
index 0498ebf..1f4581e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
@@ -19,10 +19,10 @@ package org.apache.hadoop.yarn.service.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
@@ -36,7 +36,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
@@ -60,18 +59,10 @@ public class TestBuildExternalComponents {
// 2. check component names
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
- String[] args =
- { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) };
- ClientArgs clientArgs = new ClientArgs(args);
- clientArgs.parse();
- ServiceCLI cli = new ServiceCLI() {
- @Override protected void createServiceClient() {
- client = new ServiceClient();
- client.init(conf);
- client.start();
- }
- };
- cli.exec(clientArgs);
+ AppAdminClient client = AppAdminClient.createAppAdminClient(AppAdminClient
+ .DEFAULT_TYPE, conf);
+ client.actionSave(ExampleAppJson.resourceName(appDef), null, null,
+ null);
// verify generated conf
List<Component> components =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)
Change-Id: I9213f5297a6841fab5c573e85ee4c4e5f4a0b7ff
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a28afb4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a28afb4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a28afb4
Branch: refs/heads/YARN-5881
Commit: 2a28afb4ada64a108e76bdfa99eb81a975b396a1
Parents: 2c2b7a3
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 11 10:30:23 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Sat Nov 11 16:24:03 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 31 ++
.../resource/DefaultResourceCalculator.java | 6 +
.../resource/DominantResourceCalculator.java | 7 +
.../yarn/util/resource/ResourceCalculator.java | 12 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../capacity/FifoCandidatesSelector.java | 9 +-
.../ProportionalCapacityPreemptionPolicy.java | 10 +-
.../monitor/capacity/TempQueuePerPartition.java | 16 +-
.../scheduler/AbstractResourceUsage.java | 198 +++++++
.../scheduler/QueueResourceQuotas.java | 153 ++++++
.../scheduler/ResourceUsage.java | 237 ++-------
.../scheduler/capacity/AbstractCSQueue.java | 162 +++++-
.../scheduler/capacity/CSQueue.java | 39 ++
.../scheduler/capacity/CSQueueUtils.java | 24 +-
.../CapacitySchedulerConfiguration.java | 179 ++++++-
.../scheduler/capacity/LeafQueue.java | 31 +-
.../scheduler/capacity/ParentQueue.java | 201 +++++++-
.../scheduler/capacity/UsersManager.java | 5 +-
.../PriorityUtilizationQueueOrderingPolicy.java | 11 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 15 +
.../yarn/server/resourcemanager/MockNM.java | 8 +
.../yarn/server/resourcemanager/MockRM.java | 6 +
...alCapacityPreemptionPolicyMockFramework.java | 13 +
...estProportionalCapacityPreemptionPolicy.java | 29 +-
...pacityPreemptionPolicyIntraQueueWithDRF.java | 6 +-
.../TestAbsoluteResourceConfiguration.java | 516 +++++++++++++++++++
.../capacity/TestApplicationLimits.java | 30 +-
.../TestApplicationLimitsByPartition.java | 4 +
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/capacity/TestChildQueueOrder.java | 2 +
.../scheduler/capacity/TestLeafQueue.java | 261 ++++------
.../scheduler/capacity/TestParentQueue.java | 8 +
.../scheduler/capacity/TestReservations.java | 19 +
...tPriorityUtilizationQueueOrderingPolicy.java | 3 +
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
35 files changed, 1831 insertions(+), 431 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index cda5ec7..1be8a08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1152,4 +1152,35 @@ public class StringUtils {
return s1.equalsIgnoreCase(s2);
}
+ /**
+ * <p>Checks if the String contains only unicode letters.</p>
+ *
+ * <p><code>null</code> will return <code>false</code>.
+ * An empty String (length()=0) will return <code>true</code>.</p>
+ *
+ * <pre>
+ * StringUtils.isAlpha(null) = false
+ * StringUtils.isAlpha("") = true
+ * StringUtils.isAlpha(" ") = false
+ * StringUtils.isAlpha("abc") = true
+ * StringUtils.isAlpha("ab2c") = false
+ * StringUtils.isAlpha("ab-c") = false
+ * </pre>
+ *
+ * @param str the String to check, may be null
+ * @return <code>true</code> if only contains letters, and is non-null
+ */
+ public static boolean isAlpha(String str) {
+ if (str == null) {
+ return false;
+ }
+ int sz = str.length();
+ for (int i = 0; i < sz; i++) {
+ if (Character.isLetter(str.charAt(i)) == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 7f155e7..aefa85c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -131,4 +131,10 @@ public class DefaultResourceCalculator extends ResourceCalculator {
public boolean isAnyMajorResourceZero(Resource resource) {
return resource.getMemorySize() == 0f;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6b284e3..16e4527 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -567,4 +567,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
return false;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
+ roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index d59560f..a816290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -235,4 +235,16 @@ public abstract class ResourceCalculator {
* @return returns true if any resource is zero.
*/
public abstract boolean isAnyMajorResourceZero(Resource resource);
+
+ /**
+ * Get resource <code>r</code>and normalize down using step-factor
+ * <code>stepFactor</code>.
+ *
+ * @param r
+ * resource to be multiplied
+ * @param stepFactor
+ * factor by which to normalize down
+ * @return resulting normalized resource
+ */
+ public abstract Resource normalizeDown(Resource r, Resource stepFactor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 068e7f1..0426ab7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -546,4 +546,9 @@ public class Resources {
Resource resource) {
return rc.isAnyMajorResourceZero(resource);
}
+
+ public static Resource normalizeDown(ResourceCalculator calculator,
+ Resource resource, Resource factor) {
+ return calculator.normalizeDown(resource, factor);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index f843db4..748548a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -140,10 +140,10 @@ public class FifoCandidatesSelector
// Can try preempting AMContainers (still saving atmost
// maxAMCapacityForThisQueue AMResource's) if more resources are
// required to be preemptionCandidates from this Queue.
- Resource maxAMCapacityForThisQueue = Resources.multiply(
- Resources.multiply(clusterResource,
- leafQueue.getAbsoluteCapacity()),
- leafQueue.getMaxAMResourcePerQueuePercent());
+ Resource maxAMCapacityForThisQueue = Resources
+ .multiply(
+ leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
+ leafQueue.getMaxAMResourcePerQueuePercent());
preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
@@ -199,7 +199,6 @@ public class FifoCandidatesSelector
* Given a target preemption for a specific application, select containers
* to preempt (after unreserving all reservation for that app).
*/
- @SuppressWarnings("unchecked")
private void preemptFrom(FiCaSchedulerApp app,
Resource clusterResource, Map<String, Resource> resToObtainByPartition,
List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 2c072d2..8327cb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolic
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -525,6 +526,13 @@ public class ProportionalCapacityPreemptionPolicy
float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt);
boolean preemptionDisabled = curQueue.getPreemptionDisabled();
+ QueueResourceQuotas queueResourceQuotas = curQueue
+ .getQueueResourceQuotas();
+ Resource effMinRes = queueResourceQuotas
+ .getEffectiveMinResource(partitionToLookAt);
+ Resource effMaxRes = queueResourceQuotas
+ .getEffectiveMaxResource(partitionToLookAt);
+
Resource current = Resources
.clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt));
Resource killable = Resources.none();
@@ -550,7 +558,7 @@ public class ProportionalCapacityPreemptionPolicy
ret = new TempQueuePerPartition(queueName, current, preemptionDisabled,
partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
- reserved, curQueue);
+ reserved, curQueue, effMinRes, effMaxRes);
if (curQueue instanceof ParentQueue) {
String configuredOrderingPolicy =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 89452f9..bd236fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -48,6 +48,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
double normalizedGuarantee;
+ private Resource effMinRes;
+ private Resource effMaxRes;
+
final ArrayList<TempQueuePerPartition> children;
private Collection<TempAppPerPartition> apps;
LeafQueue leafQueue;
@@ -68,7 +71,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
TempQueuePerPartition(String queueName, Resource current,
boolean preemptionDisabled, String partition, Resource killable,
float absCapacity, float absMaxCapacity, Resource totalPartitionResource,
- Resource reserved, CSQueue queue) {
+ Resource reserved, CSQueue queue, Resource effMinRes,
+ Resource effMaxRes) {
super(queueName, current, Resource.newInstance(0, 0), reserved,
Resource.newInstance(0, 0));
@@ -95,6 +99,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
this.absCapacity = absCapacity;
this.absMaxCapacity = absMaxCapacity;
this.totalPartitionResource = totalPartitionResource;
+ this.effMinRes = effMinRes;
+ this.effMaxRes = effMaxRes;
}
public void setLeafQueue(LeafQueue l) {
@@ -177,10 +183,18 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
}
public Resource getGuaranteed() {
+ if(!effMinRes.equals(Resources.none())) {
+ return Resources.clone(effMinRes);
+ }
+
return Resources.multiply(totalPartitionResource, absCapacity);
}
public Resource getMax() {
+ if(!effMaxRes.equals(Resources.none())) {
+ return Resources.clone(effMaxRes);
+ }
+
return Resources.multiply(totalPartitionResource, absMaxCapacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
new file mode 100644
index 0000000..c295323
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class AbstractResourceUsage {
+ protected ReadLock readLock;
+ protected WriteLock writeLock;
+ protected Map<String, UsageByLabel> usages;
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public AbstractResourceUsage() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ usages = new HashMap<String, UsageByLabel>();
+ usages.put(NL, new UsageByLabel(NL));
+ }
+
+ // Usage enum here to make implement cleaner
+ public enum ResourceType {
+ // CACHED_USED and CACHED_PENDING may be read by anyone, but must only
+ // be written by ordering policies
+ USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
+ 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
+ 9), EFF_MAX_RESOURCE(
+ 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+
+ private int idx;
+
+ private ResourceType(int value) {
+ this.idx = value;
+ }
+ }
+
+ public static class UsageByLabel {
+ // usage by label, contains all UsageType
+ private Resource[] resArr;
+
+ public UsageByLabel(String label) {
+ resArr = new Resource[ResourceType.values().length];
+ for (int i = 0; i < resArr.length; i++) {
+ resArr[i] = Resource.newInstance(0, 0);
+ };
+ }
+
+ public Resource getUsed() {
+ return resArr[ResourceType.USED.idx];
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{used=" + resArr[0] + "%, ");
+ sb.append("pending=" + resArr[1] + "%, ");
+ sb.append("am_used=" + resArr[2] + "%, ");
+ sb.append("reserved=" + resArr[3] + "%}");
+ sb.append("min_eff=" + resArr[9] + "%, ");
+ sb.append("max_eff=" + resArr[10] + "%}");
+ sb.append("min_effup=" + resArr[11] + "%, ");
+ return sb.toString();
+ }
+ }
+
+ private static Resource normalize(Resource res) {
+ if (res == null) {
+ return Resources.none();
+ }
+ return res;
+ }
+
+ protected Resource _get(String label, ResourceType type) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+
+ try {
+ readLock.lock();
+ UsageByLabel usage = usages.get(label);
+ if (null == usage) {
+ return Resources.none();
+ }
+ return normalize(usage.resArr[type.idx]);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ protected Resource _getAll(ResourceType type) {
+ try {
+ readLock.lock();
+ Resource allOfType = Resources.createResource(0);
+ for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
+ //all usages types are initialized
+ Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+ }
+ return allOfType;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private UsageByLabel getAndAddIfMissing(String label) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+ if (!usages.containsKey(label)) {
+ UsageByLabel u = new UsageByLabel(label);
+ usages.put(label, u);
+ return u;
+ }
+
+ return usages.get(label);
+ }
+
+ protected void _set(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.resArr[type.idx] = res;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _inc(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.addTo(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _dec(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.subtractFrom(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ readLock.lock();
+ return usages.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public Set<String> getNodePartitionsSet() {
+ try {
+ readLock.lock();
+ return usages.keySet();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
new file mode 100644
index 0000000..2e653fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+/**
+ * QueueResourceQuotas by Labels for following fields by label
+ * - EFFECTIVE_MIN_CAPACITY
+ * - EFFECTIVE_MAX_CAPACITY
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class QueueResourceQuotas extends AbstractResourceUsage {
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public QueueResourceQuotas() {
+ super();
+ }
+
+ /*
+ * Configured Minimum Resource
+ */
+ public Resource getConfiguredMinResource() {
+ return _get(NL, ResourceType.MIN_RESOURCE);
+ }
+
+ public Resource getConfiguredMinResource(String label) {
+ return _get(label, ResourceType.MIN_RESOURCE);
+ }
+
+ public void setConfiguredMinResource(String label, Resource res) {
+ _set(label, ResourceType.MIN_RESOURCE, res);
+ }
+
+ public void setConfiguredMinResource(Resource res) {
+ _set(NL, ResourceType.MIN_RESOURCE, res);
+ }
+
+ /*
+ * Configured Maximum Resource
+ */
+ public Resource getConfiguredMaxResource() {
+ return getConfiguredMaxResource(NL);
+ }
+
+ public Resource getConfiguredMaxResource(String label) {
+ return _get(label, ResourceType.MAX_RESOURCE);
+ }
+
+ public void setConfiguredMaxResource(Resource res) {
+ setConfiguredMaxResource(NL, res);
+ }
+
+ public void setConfiguredMaxResource(String label, Resource res) {
+ _set(label, ResourceType.MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResource() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public Resource getEffectiveMinResource(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public void setEffectiveMinResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ public void setEffectiveMinResource(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResource() {
+ return getEffectiveMaxResource(NL);
+ }
+
+ public Resource getEffectiveMaxResource(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE);
+ }
+
+ public void setEffectiveMaxResource(Resource res) {
+ setEffectiveMaxResource(NL, res);
+ }
+
+ public void setEffectiveMaxResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResourceUp() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public Resource getEffectiveMinResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public void setEffectiveMinResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ public void setEffectiveMinResourceUp(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResourceUp() {
+ return getEffectiveMaxResourceUp(NL);
+ }
+
+ public Resource getEffectiveMaxResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
+ }
+
+ public void setEffectiveMaxResourceUp(Resource res) {
+ setEffectiveMaxResourceUp(NL, res);
+ }
+
+ public void setEffectiveMaxResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 6f0c7d2..ede4aec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -39,63 +39,12 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*
* And it is thread-safe
*/
-public class ResourceUsage {
- private ReadLock readLock;
- private WriteLock writeLock;
- private Map<String, UsageByLabel> usages;
+public class ResourceUsage extends AbstractResourceUsage {
// short for no-label :)
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
- private final UsageByLabel usageNoLabel;
public ResourceUsage() {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- readLock = lock.readLock();
- writeLock = lock.writeLock();
-
- usages = new HashMap<String, UsageByLabel>();
- usageNoLabel = new UsageByLabel(NL);
- usages.put(NL, usageNoLabel);
- }
-
- // Usage enum here to make implement cleaner
- private enum ResourceType {
- //CACHED_USED and CACHED_PENDING may be read by anyone, but must only
- //be written by ordering policies
- USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
- CACHED_PENDING(5), AMLIMIT(6);
-
- private int idx;
-
- private ResourceType(int value) {
- this.idx = value;
- }
- }
-
- private static class UsageByLabel {
- // usage by label, contains all UsageType
- private Resource[] resArr;
-
- public UsageByLabel(String label) {
- resArr = new Resource[ResourceType.values().length];
- for (int i = 0; i < resArr.length; i++) {
- resArr[i] = Resource.newInstance(0, 0);
- };
- }
-
- public Resource getUsed() {
- return resArr[ResourceType.USED.idx];
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{used=" + resArr[0] + "%, ");
- sb.append("pending=" + resArr[1] + "%, ");
- sb.append("am_used=" + resArr[2] + "%, ");
- sb.append("reserved=" + resArr[3] + "%}");
- sb.append("am_limit=" + resArr[6] + "%, ");
- return sb.toString();
- }
+ super();
}
/*
@@ -109,22 +58,6 @@ public class ResourceUsage {
return _get(label, ResourceType.USED);
}
- public Resource getCachedUsed() {
- return _get(NL, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedUsed(String label) {
- return _get(label, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedPending() {
- return _get(NL, ResourceType.CACHED_PENDING);
- }
-
- public Resource getCachedPending(String label) {
- return _get(label, ResourceType.CACHED_PENDING);
- }
-
public void incUsed(String label, Resource res) {
_inc(label, ResourceType.USED, res);
}
@@ -145,7 +78,7 @@ public class ResourceUsage {
setUsed(NL, res);
}
- public void copyAllUsed(ResourceUsage other) {
+ public void copyAllUsed(AbstractResourceUsage other) {
try {
writeLock.lock();
for (Entry<String, UsageByLabel> entry : other.usages.entrySet()) {
@@ -160,22 +93,6 @@ public class ResourceUsage {
_set(label, ResourceType.USED, res);
}
- public void setCachedUsed(String label, Resource res) {
- _set(label, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedUsed(Resource res) {
- _set(NL, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedPending(String label, Resource res) {
- _set(label, ResourceType.CACHED_PENDING, res);
- }
-
- public void setCachedPending(Resource res) {
- _set(NL, ResourceType.CACHED_PENDING, res);
- }
-
/*
* Pending
*/
@@ -281,6 +198,47 @@ public class ResourceUsage {
_set(label, ResourceType.AMUSED, res);
}
+ public Resource getAllPending() {
+ return _getAll(ResourceType.PENDING);
+ }
+
+ public Resource getAllUsed() {
+ return _getAll(ResourceType.USED);
+ }
+
+ // Cache Used
+ public Resource getCachedUsed() {
+ return _get(NL, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedUsed(String label) {
+ return _get(label, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedPending() {
+ return _get(NL, ResourceType.CACHED_PENDING);
+ }
+
+ public Resource getCachedPending(String label) {
+ return _get(label, ResourceType.CACHED_PENDING);
+ }
+
+ public void setCachedUsed(String label, Resource res) {
+ _set(label, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedUsed(Resource res) {
+ _set(NL, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedPending(String label, Resource res) {
+ _set(label, ResourceType.CACHED_PENDING, res);
+ }
+
+ public void setCachedPending(Resource res) {
+ _set(NL, ResourceType.CACHED_PENDING, res);
+ }
+
/*
* AM-Resource Limit
*/
@@ -316,94 +274,6 @@ public class ResourceUsage {
_set(label, ResourceType.AMLIMIT, res);
}
- private static Resource normalize(Resource res) {
- if (res == null) {
- return Resources.none();
- }
- return res;
- }
-
- private Resource _get(String label, ResourceType type) {
- if (label == null || label.equals(NL)) {
- return normalize(usageNoLabel.resArr[type.idx]);
- }
- try {
- readLock.lock();
- UsageByLabel usage = usages.get(label);
- if (null == usage) {
- return Resources.none();
- }
- return normalize(usage.resArr[type.idx]);
- } finally {
- readLock.unlock();
- }
- }
-
- private Resource _getAll(ResourceType type) {
- try {
- readLock.lock();
- Resource allOfType = Resources.createResource(0);
- for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
- //all usages types are initialized
- Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
- }
- return allOfType;
- } finally {
- readLock.unlock();
- }
- }
-
- public Resource getAllPending() {
- return _getAll(ResourceType.PENDING);
- }
-
- public Resource getAllUsed() {
- return _getAll(ResourceType.USED);
- }
-
- private UsageByLabel getAndAddIfMissing(String label) {
- if (label == null || label.equals(NL)) {
- return usageNoLabel;
- }
- if (!usages.containsKey(label)) {
- UsageByLabel u = new UsageByLabel(label);
- usages.put(label, u);
- return u;
- }
-
- return usages.get(label);
- }
-
- private void _set(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- usage.resArr[type.idx] = res;
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _inc(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.addTo(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _dec(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.subtractFrom(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
public Resource getCachedDemand(String label) {
try {
readLock.lock();
@@ -415,23 +285,4 @@ public class ResourceUsage {
readLock.unlock();
}
}
-
- @Override
- public String toString() {
- try {
- readLock.lock();
- return usages.toString();
- } finally {
- readLock.unlock();
- }
- }
-
- public Set<String> getNodePartitionsSet() {
- try {
- readLock.lock();
- return usages.keySet();
- } finally {
- readLock.unlock();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 183cb36..b2539c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -87,6 +89,7 @@ public abstract class AbstractCSQueue implements CSQueue {
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
+ Set<String> resourceTypes;
final RMNodeLabelsManager labelManager;
String defaultLabelExpression;
@@ -102,6 +105,14 @@ public abstract class AbstractCSQueue implements CSQueue {
// etc.
QueueCapacities queueCapacities;
+ QueueResourceQuotas queueResourceQuotas;
+
+ protected enum CapacityConfigType {
+ NONE, PERCENTAGE, ABSOLUTE_RESOURCE
+ };
+ protected CapacityConfigType capacityConfigType =
+ CapacityConfigType.NONE;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
@@ -141,6 +152,9 @@ public abstract class AbstractCSQueue implements CSQueue {
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
+ // initialize queueResourceQuotas
+ queueResourceQuotas = new QueueResourceQuotas();
+
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
@@ -276,6 +290,10 @@ public abstract class AbstractCSQueue implements CSQueue {
this.defaultLabelExpression =
csContext.getConfiguration().getDefaultNodeLabelExpression(
getQueuePath());
+ this.resourceTypes = new HashSet<String>();
+ for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
+ resourceTypes.add(type.toString().toLowerCase());
+ }
// inherit from parent if labels not set
if (this.accessibleLabels == null && parent != null) {
@@ -292,6 +310,11 @@ public abstract class AbstractCSQueue implements CSQueue {
// After we setup labels, we can setup capacities
setupConfigurableCapacities();
+ // Also fetch minimum/maximum resource constraint for this queue if
+ // configured.
+ capacityConfigType = CapacityConfigType.NONE;
+ updateConfigurableResourceRequirement(getQueuePath(), clusterResource);
+
this.maximumAllocation =
csContext.getConfiguration().getMaximumAllocationPerQueue(
getQueuePath());
@@ -364,6 +387,125 @@ public abstract class AbstractCSQueue implements CSQueue {
return unionInheritedWeights;
}
+ protected void updateConfigurableResourceRequirement(String queuePath,
+ Resource clusterResource) {
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ Set<String> configuredNodelabels = conf.getConfiguredNodeLabels(queuePath);
+
+ for (String label : configuredNodelabels) {
+ Resource minResource = conf.getMinimumResourceRequirement(label,
+ queuePath, resourceTypes);
+ Resource maxResource = conf.getMaximumResourceRequirement(label,
+ queuePath, resourceTypes);
+
+ if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
+ this.capacityConfigType = (!minResource.equals(Resources.none())
+ && queueCapacities.getAbsoluteCapacity(label) == 0f)
+ ? CapacityConfigType.ABSOLUTE_RESOURCE
+ : CapacityConfigType.PERCENTAGE;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is updated as '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
+ }
+
+ validateAbsoluteVsPercentageCapacityConfig(minResource);
+
+ // If min resource for a resource type is greater than its max resource,
+ // throw exception to handle such error configs.
+ if (!maxResource.equals(Resources.none()) && Resources.greaterThan(
+ resourceCalculator, clusterResource, minResource, maxResource)) {
+ throw new IllegalArgumentException("Min resource configuration "
+ + minResource + " is greater than its max value:" + maxResource
+ + " in queue:" + getQueueName());
+ }
+
+ // If parent's max resource is lesser to a specific child's max
+ // resource, throw exception to handle such error configs.
+ if (parent != null) {
+ Resource parentMaxRes = parent.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ parentMaxRes, Resources.none())) {
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ maxResource, parentMaxRes)) {
+ throw new IllegalArgumentException("Max resource configuration "
+ + maxResource + " is greater than parents max value:"
+ + parentMaxRes + " in queue:" + getQueueName());
+ }
+
+ // If child's max resource is not set, but its parent max resource is
+ // set, we must set child max resource to its parent's.
+ if (maxResource.equals(Resources.none())
+ && !minResource.equals(Resources.none())) {
+ maxResource = Resources.clone(parentMaxRes);
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating absolute resource configuration for queue:"
+ + getQueueName() + " as minResource=" + minResource
+ + " and maxResource=" + maxResource);
+ }
+
+ queueResourceQuotas.setConfiguredMinResource(label, minResource);
+ queueResourceQuotas.setConfiguredMaxResource(label, maxResource);
+ }
+ }
+
+ private void validateAbsoluteVsPercentageCapacityConfig(
+ Resource minResource) {
+ CapacityConfigType localType = CapacityConfigType.PERCENTAGE;
+ if (!minResource.equals(Resources.none())) {
+ localType = CapacityConfigType.ABSOLUTE_RESOURCE;
+ }
+
+ if (!queueName.equals("root")
+ && !this.capacityConfigType.equals(localType)) {
+ throw new IllegalArgumentException("Queue '" + getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource.");
+ }
+ }
+
+ @Override
+ public CapacityConfigType getCapacityConfigType() {
+ return capacityConfigType;
+ }
+
+ @Override
+ public Resource getEffectiveCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityUp(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMinResource(label),
+ minimumAllocation);
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMaxResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMaxResource(label),
+ minimumAllocation);
+ }
+
private void initializeQueueState(QueueState previousState,
QueueState configuredState, QueueState parentState) {
// verify that we can not any value for State other than RUNNING/STOPPED
@@ -555,6 +697,11 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
+ public QueueResourceQuotas getQueueResourceQuotas() {
+ return queueResourceQuotas;
+ }
+
+ @Override
public ReentrantReadWriteLock.ReadLock getReadLock() {
return readLock;
}
@@ -604,7 +751,7 @@ public abstract class AbstractCSQueue implements CSQueue {
* limit-set-by-parent)
*/
Resource queueMaxResource =
- getQueueMaxResource(nodePartition, clusterResource);
+ getQueueMaxResource(nodePartition);
return Resources.min(resourceCalculator, clusterResource,
queueMaxResource, currentResourceLimits.getLimit());
@@ -617,11 +764,8 @@ public abstract class AbstractCSQueue implements CSQueue {
return Resources.none();
}
- Resource getQueueMaxResource(String nodePartition, Resource clusterResource) {
- return Resources.multiplyAndNormalizeDown(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- queueCapacities.getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource getQueueMaxResource(String nodePartition) {
+ return getEffectiveMaxCapacity(nodePartition);
}
public boolean hasChildQueues() {
@@ -782,7 +926,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.incUsed(nodeLabel, resourceToInc);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.incUsedResource(nodeLabel, resourceToInc, null);
}
@@ -798,7 +942,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodeLabel, resourceToDec);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.decUsedResource(nodeLabel, resourceToDec, null);
}
@@ -904,7 +1048,7 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResourceLimit;
if (allocation.getSchedulingMode()
== SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
- maxResourceLimit = getQueueMaxResource(partition, cluster);
+ maxResourceLimit = getQueueMaxResource(partition);
} else{
maxResourceLimit = labelManager.getResourceByLabel(
schedulerContainer.getNodePartition(), cluster);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 43e7f53..2e29a71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -41,10 +41,12 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -355,4 +357,41 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return map of usernames and corresponding weight
*/
Map<String, Float> getUserWeights();
+
+ /**
+ * Get QueueResourceQuotas associated with each queue.
+ * @return QueueResourceQuotas
+ */
+ public QueueResourceQuotas getQueueResourceQuotas();
+
+ /**
+ * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+ * @return CapacityConfigType
+ */
+ public CapacityConfigType getCapacityConfigType();
+
+ /**
+ * Get effective capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective queue capacity
+ */
+ Resource getEffectiveCapacity(String label);
+ Resource getEffectiveCapacityUp(String label);
+ Resource getEffectiveCapacityDown(String label, Resource factor);
+
+ /**
+ * Get effective max capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective max queue capacity
+ */
+ Resource getEffectiveMaxCapacity(String label);
+ Resource getEffectiveMaxCapacityDown(String label, Resource factor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e1014c1..81dec80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -150,7 +150,7 @@ class CSQueueUtils {
}
}
}
-
+
// Set absolute capacities for {capacity, maximum-capacity}
private static void updateAbsoluteCapacitiesByNodeLabels(
QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
@@ -180,8 +180,8 @@ class CSQueueUtils {
* used resource for all partitions of this queue.
*/
public static void updateUsedCapacity(final ResourceCalculator rc,
- final Resource totalPartitionResource, String nodePartition,
- AbstractCSQueue childQueue) {
+ final Resource totalPartitionResource, Resource clusterResource,
+ String nodePartition, AbstractCSQueue childQueue) {
QueueCapacities queueCapacities = childQueue.getQueueCapacities();
CSQueueMetrics queueMetrics = childQueue.getMetrics();
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -193,11 +193,8 @@ class CSQueueUtils {
if (Resources.greaterThan(rc, totalPartitionResource,
totalPartitionResource, Resources.none())) {
- // queueGuaranteed = totalPartitionedResource *
- // absolute_capacity(partition)
- Resource queueGuranteedResource =
- Resources.multiply(totalPartitionResource,
- queueCapacities.getAbsoluteCapacity(nodePartition));
+ Resource queueGuranteedResource = childQueue
+ .getEffectiveCapacity(nodePartition);
// make queueGuranteed >= minimum_allocation to avoid divided by 0.
queueGuranteedResource =
@@ -248,9 +245,7 @@ class CSQueueUtils {
for (String partition : nodeLabels) {
// Calculate guaranteed resource for a label in a queue by below logic.
// (total label resource) * (absolute capacity of label in that queue)
- Resource queueGuranteedResource = Resources.multiply(nlm
- .getResourceByLabel(partition, cluster), queue.getQueueCapacities()
- .getAbsoluteCapacity(partition));
+ Resource queueGuranteedResource = queue.getEffectiveCapacity(partition);
// Available resource in queue for a specific label will be calculated as
// {(guaranteed resource for a label in a queue) -
@@ -289,15 +284,14 @@ class CSQueueUtils {
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
if (nodePartition == null) {
- for (String partition : Sets.union(
- queueCapacities.getNodePartitionsSet(),
+ for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
queueResourceUsage.getNodePartitionsSet())) {
updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
- partition, childQueue);
+ cluster, partition, childQueue);
}
} else {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
- nodePartition, childQueue);
+ cluster, nodePartition, childQueue);
}
// Update queue metrics w.r.t node labels. In a generic way, we can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index bfead35..40de74d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -60,6 +61,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.Set;
import java.util.StringTokenizer;
@@ -317,6 +320,21 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
+ /** Configuring absolute min/max resources in a queue **/
+ @Private
+ public static final String MINIMUM_RESOURCE = "min-resource";
+
+ @Private
+ public static final String MAXIMUM_RESOURCE = "max-resource";
+
+ public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
+
+ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+
+ public enum AbsoluteResourceType {
+ MEMORY, VCORES;
+ }
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
@@ -394,7 +412,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public float getNonLabeledQueueCapacity(String queue) {
float capacity = queue.equals("root") ? 100.0f : getFloat(
- getQueuePrefix(queue) + CAPACITY, UNDEFINED);
+ getQueuePrefix(queue) + CAPACITY, 0f);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
@@ -1523,4 +1541,163 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
setLong(getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, defaultLifetime);
}
+
+ public static String getUnits(String resourceValue) {
+ String units;
+ for (int i = 0; i < resourceValue.length(); i++) {
+ if (Character.isAlphabetic(resourceValue.charAt(i))) {
+ units = resourceValue.substring(i);
+ if (StringUtils.isAlpha(units)) {
+ return units;
+ }
+ }
+ }
+ return "";
+ }
+
+ /**
+ * Get absolute minimum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return ResourceInformation
+ */
+ public Resource getMinimumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MINIMUM_RESOURCE);
+ }
+
+ /**
+ * Get absolute maximum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return Resource
+ */
+ public Resource getMaximumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MAXIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMinimumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMaximumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+ }
+
+ private void updateMinMaxResourceToConf(String label, String queue,
+ Resource resource, String type) {
+ if (queue.equals("root")) {
+ throw new IllegalArgumentException(
+ "Cannot set resource, root queue will take 100% of cluster capacity");
+ }
+
+ StringBuilder resourceString = new StringBuilder();
+ resourceString
+ .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
+ + resource.getMemorySize() + ","
+ + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
+ + resource.getVirtualCores() + "]");
+
+ String prefix = getQueuePrefix(queue) + type;
+ if (!label.isEmpty()) {
+ prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+ + DOT + type;
+ }
+ set(prefix, resourceString.toString());
+ }
+
+ private Resource internalGetLabeledResourceRequirementForQueue(String queue,
+ String label, Set<String> resourceTypes, String suffix) {
+ String propertyName = getNodeLabelPrefix(queue, label) + suffix;
+ String resourceString = get(propertyName);
+ if (resourceString == null || resourceString.isEmpty()) {
+ return Resources.none();
+ }
+
+ // Define resource here.
+ Resource resource = Resource.newInstance(0l, 0);
+ Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
+ .matcher(resourceString);
+ /*
+ * Absolute resource configuration for a queue will be grouped by "[]".
+ * Syntax of absolute resource config could be like below
+ * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores".
+ */
+ if (matcher.find()) {
+ // Get the sub-group.
+ String subGroup = matcher.group(1);
+ if (subGroup.trim().isEmpty()) {
+ return Resources.none();
+ }
+
+ for (String kvPair : subGroup.trim().split(",")) {
+ String[] splits = kvPair.split("=");
+
+ // Ensure that each sub string is key value pair separated by '='.
+ if (splits != null && splits.length > 1) {
+ updateResourceValuesFromConfig(resourceTypes, resource, splits);
+ }
+ }
+ }
+
+ // Memory has to be configured always.
+ if (resource.getMemorySize() == 0l) {
+ return Resources.none();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CSConf - getAbsolueResourcePerQueue: prefix="
+ + getNodeLabelPrefix(queue, label) + ", capacity=" + resource);
+ }
+ return resource;
+ }
+
+ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
+ Resource resource, String[] splits) {
+
+ // If key is not a valid type, skip it.
+ if (!resourceTypes.contains(splits[0])) {
+ return;
+ }
+
+ String units = getUnits(splits[1]);
+ Long resourceValue = Long
+ .valueOf(splits[1].substring(0, splits[1].length() - units.length()));
+
+ // Convert all incoming units to MB if units is configured.
+ if (!units.isEmpty()) {
+ resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
+ }
+
+ // map it based on key.
+ AbsoluteResourceType resType = AbsoluteResourceType
+ .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+ switch (resType) {
+ case MEMORY :
+ resource.setMemorySize(resourceValue);
+ break;
+ case VCORES :
+ resource.setVirtualCores(resourceValue.intValue());
+ break;
+ default :
+ break;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index e8342d9..1460121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,12 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = Resources
- .multiplyAndNormalizeUp(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition,
- lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -705,11 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
@@ -965,6 +956,14 @@ public class LeafQueue extends AbstractCSQueue {
private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
// Set preemption-allowed:
// For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
+ if (!queueResourceQuotas.getEffectiveMinResource(nodePartition)
+ .equals(Resources.none())) {
+ limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
+ csContext.getClusterResource(), queueUsage.getUsed(nodePartition),
+ queueResourceQuotas.getEffectiveMinResource(nodePartition)));
+ return;
+ }
+
float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
@@ -1344,7 +1343,7 @@ public class LeafQueue extends AbstractCSQueue {
currentPartitionResourceLimit =
partition.equals(RMNodeLabelsManager.NO_LABEL)
? currentPartitionResourceLimit
- : getQueueMaxResource(partition, clusterResource);
+ : getQueueMaxResource(partition);
Resource headroom = Resources.componentwiseMin(
Resources.subtract(userLimitResource, user.getUsed(partition)),
@@ -1716,12 +1715,8 @@ public class LeafQueue extends AbstractCSQueue {
// this. So need cap limits by queue's max capacity here.
this.cachedResourceLimitsForHeadroom =
new ResourceLimits(currentResourceLimits.getLimit());
- Resource queueMaxResource =
- Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
- .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
- queueCapacities
- .getAbsoluteMaximumCapacity(RMNodeLabelsManager.NO_LABEL),
- minimumAllocation);
+ Resource queueMaxResource = getEffectiveMaxCapacityDown(
+ RMNodeLabelsManager.NO_LABEL, minimumAllocation);
this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
resourceCalculator, clusterResource, queueMaxResource,
currentResourceLimits.getLimit()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index d61951b..d5a1354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -162,31 +164,78 @@ public class ParentQueue extends AbstractCSQueue {
writeLock.lock();
// Validate
float childCapacities = 0;
+ Resource minResDefaultLabel = Resources.createResource(0, 0);
for (CSQueue queue : childQueues) {
childCapacities += queue.getCapacity();
+ Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource());
+
+ // If any child queue is using percentage based capacity model vs parent
+ // queues' absolute configuration or vice versa, throw back an
+ // exception.
+ if (!queueName.equals("root") && getCapacity() != 0f
+ && !queue.getQueueResourceQuotas().getConfiguredMinResource()
+ .equals(Resources.none())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource together.");
+ }
}
+
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
- (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
- throw new IllegalArgumentException(
- "Illegal" + " capacity of " + childCapacities
- + " for children of queue " + queueName);
+ if ((minResDefaultLabel.equals(Resources.none())
+ && (queueCapacities.getCapacity() > 0) && (delta > PRECISION))
+ || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+ throw new IllegalArgumentException("Illegal" + " capacity of "
+ + childCapacities + " for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
+ Resource minRes = Resources.createResource(0, 0);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+ scheduler.getClusterResource());
for (CSQueue queue : childQueues) {
sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+
+ // If any child queue of a label is using percentage based capacity
+ // model vs parent queues' absolute configuration or vice versa, throw
+ // back an exception
+ if (!queueName.equals("root") && !this.capacityConfigType
+ .equals(queue.getCapacityConfigType())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + "configuration or absolute resource together for label:"
+ + nodeLabel);
+ }
+
+ // Accumulate all min/max resource configured for all child queues.
+ Resources.addTo(minRes, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource(nodeLabel));
}
- if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+ if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0
+ && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
throw new IllegalArgumentException(
"Illegal" + " capacity of " + sum + " for children of queue "
+ queueName + " for label=" + nodeLabel);
}
+
+ // Ensure that for each parent queue: parent.min-resource >=
+ // Σ(child.min-resource).
+ Resource parentMinResource = queueResourceQuotas
+ .getConfiguredMinResource(nodeLabel);
+ if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
+ resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
+ throw new IllegalArgumentException("Parent Queues" + " capacity: "
+ + parentMinResource + " is less than" + " to its children:"
+ + minRes + " for queue:" + queueName);
+ }
}
this.childQueues.clear();
@@ -687,11 +736,8 @@ public class ParentQueue extends AbstractCSQueue {
child.getQueueResourceUsage().getUsed(nodePartition));
// Get child's max resource
- Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
+ nodePartition, minimumAllocation);
// Child's limit should be capped by child configured max resource
childLimit =
@@ -827,6 +873,14 @@ public class ParentQueue extends AbstractCSQueue {
ResourceLimits resourceLimits) {
try {
writeLock.lock();
+
+ // Update effective capacity in all parent queue.
+ Set<String> configuredNodelabels = csContext.getConfiguration()
+ .getConfiguredNodeLabels(getQueuePath());
+ for (String label : configuredNodelabels) {
+ calculateEffectiveResourcesAndCapacity(label, clusterResource);
+ }
+
// Update all children
for (CSQueue childQueue : childQueues) {
// Get ResourceLimits of child queue before assign containers
@@ -848,6 +902,110 @@ public class ParentQueue extends AbstractCSQueue {
return true;
}
+ private void calculateEffectiveResourcesAndCapacity(String label,
+ Resource clusterResource) {
+
+ // For root queue, ensure that max/min resource is updated to latest
+ // cluster resource.
+ Resource resourceByLabel = labelManager.getResourceByLabel(label,
+ clusterResource);
+ if (getQueueName().equals("root")) {
+ queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel);
+ queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ }
+
+ // Total configured min resources of direct children of queue
+ Resource configuredMinResources = Resource.newInstance(0L, 0);
+ for (CSQueue childQueue : getChildQueues()) {
+ Resources.addTo(configuredMinResources,
+ childQueue.getQueueResourceQuotas().getConfiguredMinResource(label));
+ }
+
+ // Factor to scale down effective resource: When cluster has sufficient
+ // resources, effective_min_resources will be same as configured
+ // min_resources.
+ float effectiveMinRatio = 1;
+ ResourceCalculator rc = this.csContext.getResourceCalculator();
+ if (getQueueName().equals("root")) {
+ if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
+ clusterResource, resourceByLabel, configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ resourceByLabel, configuredMinResources);
+ }
+ } else {
+ if (Resources.lessThan(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources);
+ }
+ }
+
+ // loop and do this for all child queues
+ for (CSQueue childQueue : getChildQueues()) {
+ Resource minResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMinResource(label);
+
+ // Update effective resource (min/max) to each child queue.
+ if (childQueue.getCapacityConfigType()
+ .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(minResource, effectiveMinRatio));
+
+ // Max resource of a queue should be a minimum of {configuredMaxRes,
+ // parentMaxRes}. parentMaxRes could be configured value. But if not
+ // present could also be taken from effective max resource of parent.
+ Resource parentMaxRes = queueResourceQuotas
+ .getConfiguredMaxResource(label);
+ if (parentMaxRes.equals(Resources.none())) {
+ parentMaxRes = parent.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label);
+ }
+
+ // Minimum of {childMaxResource, parentMaxRes}. However if
+ // childMaxResource is empty, consider parent's max resource alone.
+ Resource childMaxResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ Resource effMaxResource = Resources.min(resourceCalculator,
+ resourceByLabel, childMaxResource.equals(Resources.none())
+ ? parentMaxRes
+ : childMaxResource,
+ parentMaxRes);
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.clone(effMaxResource));
+ } else {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label)));
+
+ childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label),
+ minimumAllocation));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc,
+ resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label),
+ minimumAllocation));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating effective min resource for queue:"
+ + childQueue.getQueueName() + " as effMinResource="
+ + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label)
+ + "and Updating effective max resource as effMaxResource="
+ + childQueue.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label));
+ }
+ }
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
@@ -980,9 +1138,21 @@ public class ParentQueue extends AbstractCSQueue {
* When this happens, we have to preempt killable container (on same or different
* nodes) of parent queue to avoid violating parent's max resource.
*/
- if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
- < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
- killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+ .equals(Resources.none())) {
+ if (Resources.lessThan(resourceCalculator, clusterResource,
+ queueResourceQuotas.getEffectiveMaxResource(nodePartition),
+ queueUsage.getUsed(nodePartition))) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
+ } else {
+ if (getQueueCapacities()
+ .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities()
+ .getAbsoluteUsedCapacity(nodePartition)) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
}
} finally {
writeLock.unlock();
@@ -999,8 +1169,7 @@ public class ParentQueue extends AbstractCSQueue {
Resource partitionResource = labelManager.getResourceByLabel(partition,
null);
- Resource maxResource = Resources.multiply(partitionResource,
- getQueueCapacities().getAbsoluteMaximumCapacity(partition));
+ Resource maxResource = getEffectiveMaxCapacity(partition);
while (Resources.greaterThan(resourceCalculator, partitionResource,
queueUsage.getUsed(partition), maxResource)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a28afb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 33f30b0..efc20e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,10 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = Resources.multiplyAndNormalizeUp(
- resourceCalculator, partitionResource,
- lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition),
- lQueue.getMinimumAllocation());
+ Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN
related params in the service REST API. Contributed by Jian He
Posted by su...@apache.org.
YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74
Branch: refs/heads/YARN-5881
Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02
Parents: a288b0a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Fri Nov 3 11:25:57 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../definition/YARN-Services-Examples.md | 8 +-
...RN-Simplified-V1-API-Layer-For-Services.yaml | 6 +-
.../examples/httpd-no-dns/httpd-no-dns.json | 6 +-
.../examples/httpd/httpd.json | 4 +-
.../hadoop/yarn/service/ServiceMaster.java | 6 +
.../hadoop/yarn/service/ServiceMetrics.java | 4 -
.../hadoop/yarn/service/ServiceScheduler.java | 15 --
.../yarn/service/api/ServiceApiConstants.java | 4 +-
.../yarn/service/api/records/ConfigFile.java | 32 ++--
.../yarn/service/api/records/Configuration.java | 4 +-
.../service/api/records/ReadinessCheck.java | 24 +--
.../yarn/service/client/ServiceClient.java | 61 ++++---
.../yarn/service/component/Component.java | 29 ++--
.../component/instance/ComponentInstance.java | 37 +++--
.../yarn/service/conf/YarnServiceConf.java | 14 +-
.../yarn/service/conf/YarnServiceConstants.java | 2 +
.../service/monitor/probe/MonitorUtils.java | 4 +-
.../provider/AbstractClientProvider.java | 17 +-
.../yarn/service/provider/ProviderUtils.java | 12 +-
.../timelineservice/ServiceTimelineEvent.java | 4 +-
.../ServiceTimelinePublisher.java | 48 +++---
.../yarn/service/conf/TestAppJsonResolve.java | 8 +-
.../TestServiceTimelinePublisher.java | 2 +-
.../service/conf/examples/app-override.json | 6 +-
.../markdown/yarn-service/Configurations.md | 165 +++++++++++++++++++
.../src/site/markdown/yarn-service/Examples.md | 4 +-
.../src/site/markdown/yarn-service/Overview.md | 3 +-
.../markdown/yarn-service/YarnServiceAPI.md | 14 +-
.../app/controllers/yarn-app/components.js | 2 +-
.../app/templates/components/metrics-table.hbs | 2 +-
.../templates/yarn-component-instance/info.hbs | 4 +-
.../templates/yarn-component-instances/info.hbs | 2 +-
32 files changed, 384 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
index 3cd3d48..00b21dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -176,14 +176,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -216,14 +216,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index cc76259..088b50c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -340,7 +340,7 @@ definitions:
enum:
- HTTP
- PORT
- props:
+ properties:
type: object
description: A blob of key value pairs that will be used to configure the check.
additionalProperties:
@@ -353,7 +353,7 @@ definitions:
properties:
properties:
type: object
- description: A blob of key-value pairs of common service properties.
+ description: A blob of key-value pairs for configuring the YARN service AM
additionalProperties:
type: string
env:
@@ -386,7 +386,7 @@ definitions:
src_file:
type: string
description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.
- props:
+ properties:
type: object
description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.
Container:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
index 6b35538..7bf2f68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -16,16 +16,16 @@
},
"readiness_check": {
"type": "HTTP",
- "props": {
+ "properties": {
"url": "http://${THIS_HOST}:8080"
}
},
"configuration": {
"files": [
{
- "type": "ENV",
+ "type": "TEMPLATE",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
index e63376d..8786214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd/httpd.json
@@ -17,9 +17,9 @@
"configuration": {
"files": [
{
- "type": "ENV",
+ "type": "TEMPLATE",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index d58e32b..2d86a5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -78,6 +78,12 @@ public class ServiceMaster extends CompositeService {
fs.setAppDir(appDir);
loadApplicationJson(context, fs);
+ // Take yarn config from YarnFile and merge them into YarnConfiguration
+ for (Map.Entry<String, String> entry : context.service
+ .getConfiguration().getProperties().entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+
ContainerId amContainerId = getAMContainerId();
ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
index 9fc886e..b5bbb7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMetrics.java
@@ -35,9 +35,6 @@ public class ServiceMetrics implements MetricsSource {
@Metric("containers requested")
public MutableGaugeInt containersRequested;
- @Metric("anti-affinity containers pending")
- public MutableGaugeInt pendingAAContainers;
-
@Metric("containers running")
public MutableGaugeInt containersRunning;
@@ -86,7 +83,6 @@ public class ServiceMetrics implements MetricsSource {
@Override public String toString() {
return "ServiceMetrics{"
+ "containersRequested=" + containersRequested.value()
- + ", pendingAAContainers=" + pendingAAContainers.value()
+ ", containersRunning=" + containersRunning.value()
+ ", containersDesired=" + containersDesired.value()
+ ", containersSucceeded=" + containersSucceeded.value()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 631f89e..a7b7e22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -397,10 +397,6 @@ public class ServiceScheduler extends CompositeService {
PersistencePolicies.APPLICATION);
serviceRecord.description = "YarnServiceMaster";
- // set any provided attributes
- setUserProvidedServiceRecordAttributes(service.getConfiguration(),
- serviceRecord);
-
executorService.submit(new Runnable() {
@Override public void run() {
try {
@@ -426,17 +422,6 @@ public class ServiceScheduler extends CompositeService {
}
}
- private void setUserProvidedServiceRecordAttributes(
- org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) {
- String prefix = "service.record.attribute";
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- if (entry.getKey().startsWith(prefix)) {
- String key = entry.getKey().substring(prefix.length() + 1);
- record.set(key, entry.getValue().trim());
- }
- }
- }
-
private void createAllComponents() {
long allocateId = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
index a85191c..a4f2243 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/ServiceApiConstants.java
@@ -53,9 +53,9 @@ public interface ServiceApiConstants {
String CONTAINER_ID = $("CONTAINER_ID");
// Templates for component instance host/IP
- String COMPONENT_HOST = $("%s_HOST");
+ String COMPONENT_INSTANCE_HOST = $("%s_HOST");
- String COMPONENT_IP = $("%s_IP");
+ String COMPONENT_INSTANCE_IP = $("%s_IP");
// Constants for default cluster ZK
String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
index fb088e2..984e6f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
@@ -49,13 +49,13 @@ public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
/**
- * Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
+ * Config Type. XML, JSON, YAML, TEMPLATE and HADOOP_XML are supported.
**/
@XmlType(name = "config_type")
@XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
- "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;
+ "TEMPLATE"), HADOOP_XML("HADOOP_XML"),;
private String value;
@@ -73,15 +73,15 @@ public class ConfigFile implements Serializable {
private TypeEnum type = null;
private String destFile = null;
private String srcFile = null;
- private Map<String, String> props = new HashMap<>();
+ private Map<String, String> properties = new HashMap<>();
public ConfigFile copy() {
ConfigFile copy = new ConfigFile();
copy.setType(this.getType());
copy.setSrcFile(this.getSrcFile());
copy.setDestFile(this.getDestFile());
- if (this.getProps() != null && !this.getProps().isEmpty()) {
- copy.getProps().putAll(this.getProps());
+ if (this.getProperties() != null && !this.getProperties().isEmpty()) {
+ copy.getProperties().putAll(this.getProperties());
}
return copy;
}
@@ -156,28 +156,28 @@ public class ConfigFile implements Serializable {
in the dest_file and these properties will overwrite, if any, existing
properties in src_file or be added as new properties in src_file.
**/
- public ConfigFile props(Map<String, String> props) {
- this.props = props;
+ public ConfigFile properties(Map<String, String> properties) {
+ this.properties = properties;
return this;
}
@ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type."
+ " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any,"
+ " existing properties in src_file or be added as new properties in src_file.")
- @JsonProperty("props")
- public Map<String, String> getProps() {
- return props;
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
}
- public void setProps(Map<String, String> props) {
- this.props = props;
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
}
public long getLong(String name, long defaultValue) {
if (name == null) {
return defaultValue;
}
- String value = props.get(name.trim());
+ String value = properties.get(name.trim());
return Long.parseLong(value);
}
@@ -185,7 +185,7 @@ public class ConfigFile implements Serializable {
if (name == null) {
return defaultValue;
}
- return Boolean.valueOf(props.get(name.trim()));
+ return Boolean.valueOf(properties.get(name.trim()));
}
@Override
@@ -204,7 +204,7 @@ public class ConfigFile implements Serializable {
@Override
public int hashCode() {
- return Objects.hash(type, destFile, srcFile, props);
+ return Objects.hash(type, destFile, srcFile, properties);
}
@Override
@@ -215,7 +215,7 @@ public class ConfigFile implements Serializable {
sb.append(" type: ").append(toIndentedString(type)).append("\n");
sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
sb.append("}");
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index 2f8ca96..dc6dff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -212,8 +212,8 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProps(),
- thatFile.getProps());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+ thatFile.getProperties());
thatMap.remove(thisFile.getDestFile());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index b25828f..0a3713c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -69,7 +69,7 @@ public class ReadinessCheck implements Serializable {
}
private TypeEnum type = null;
- private Map<String, String> props = new HashMap<String, String>();
+ private Map<String, String> properties = new HashMap<String, String>();
private Artifact artifact = null;
/**
@@ -91,27 +91,27 @@ public class ReadinessCheck implements Serializable {
this.type = type;
}
- public ReadinessCheck props(Map<String, String> props) {
- this.props = props;
+ public ReadinessCheck properties(Map<String, String> properties) {
+ this.properties = properties;
return this;
}
public ReadinessCheck putPropsItem(String key, String propsItem) {
- this.props.put(key, propsItem);
+ this.properties.put(key, propsItem);
return this;
}
/**
* A blob of key value pairs that will be used to configure the check.
- * @return props
+ * @return properties
**/
@ApiModelProperty(example = "null", value = "A blob of key value pairs that will be used to configure the check.")
- public Map<String, String> getProps() {
- return props;
+ public Map<String, String> getProperties() {
+ return properties;
}
- public void setProps(Map<String, String> props) {
- this.props = props;
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
}
/**
@@ -148,13 +148,13 @@ public class ReadinessCheck implements Serializable {
}
ReadinessCheck readinessCheck = (ReadinessCheck) o;
return Objects.equals(this.type, readinessCheck.type) &&
- Objects.equals(this.props, readinessCheck.props) &&
+ Objects.equals(this.properties, readinessCheck.properties) &&
Objects.equals(this.artifact, readinessCheck.artifact);
}
@Override
public int hashCode() {
- return Objects.hash(type, props, artifact);
+ return Objects.hash(type, properties, artifact);
}
@@ -164,7 +164,7 @@ public class ReadinessCheck implements Serializable {
sb.append("class ReadinessCheck {\n");
sb.append(" type: ").append(toIndentedString(type)).append("\n");
- sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append(" properties: ").append(toIndentedString(properties)).append("\n");
sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
sb.append("}");
return sb.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 636b127..775a642 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -38,17 +38,8 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
@@ -99,7 +90,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@@ -553,8 +544,11 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime());
submissionContext.setApplicationTimeouts(appTimeout);
}
- submissionContext.setMaxAppAttempts(conf.getInt(
- YarnServiceConf.AM_RESTART_MAX, 2));
+ submissionContext.setMaxAppAttempts(YarnServiceConf
+ .getInt(YarnServiceConf.AM_RESTART_MAX, 20, app.getConfiguration(),
+ conf));
+
+ setLogAggregationContext(app, conf, submissionContext);
Map<String, LocalResource> localResources = new HashMap<>();
@@ -568,14 +562,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (LOG.isDebugEnabled()) {
printLocalResources(localResources);
}
- Map<String, String> env = addAMEnv(conf);
+ Map<String, String> env = addAMEnv();
// create AM CLI
- String cmdStr =
- buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
+ String cmdStr = buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
- .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM,
- app.getConfiguration(), conf), 1));
+ .getLong(YarnServiceConf.AM_RESOURCE_MEM,
+ YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, app.getConfiguration(),
+ conf), 1));
String queue = app.getQueue();
if (StringUtils.isEmpty(queue)) {
queue = conf.get(YARN_QUEUE, "default");
@@ -598,6 +592,33 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return submissionContext.getApplicationId();
}
+ private void setLogAggregationContext(Service app, Configuration conf,
+ ApplicationSubmissionContext submissionContext) {
+ LogAggregationContext context = Records.newRecord(LogAggregationContext
+ .class);
+ String finalLogInclude = YarnServiceConf.get
+ (FINAL_LOG_INCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(finalLogInclude)) {
+ context.setIncludePattern(finalLogInclude);
+ }
+ String finalLogExclude = YarnServiceConf.get
+ (FINAL_LOG_EXCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(finalLogExclude)) {
+ context.setExcludePattern(finalLogExclude);
+ }
+ String rollingLogInclude = YarnServiceConf.get
+ (ROLLING_LOG_INCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(rollingLogInclude)) {
+ context.setRolledLogsIncludePattern(rollingLogInclude);
+ }
+ String rollingLogExclude = YarnServiceConf.get
+ (ROLLING_LOG_EXCLUSION_PATTERN, null, app.getConfiguration(), conf);
+ if (!StringUtils.isEmpty(rollingLogExclude)) {
+ context.setRolledLogsExcludePattern(rollingLogExclude);
+ }
+ submissionContext.setLogAggregationContext(context);
+ }
+
private void printLocalResources(Map<String, LocalResource> map) {
LOG.debug("Added LocalResource for localization: ");
StringBuilder builder = new StringBuilder();
@@ -635,7 +656,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return cmdStr;
}
- private Map<String, String> addAMEnv(Configuration conf) throws IOException {
+ private Map<String, String> addAMEnv() throws IOException {
Map<String, String> env = new HashMap<>();
ClasspathConstructor classpath =
buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index a5dd39c..7667a53 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -116,7 +116,7 @@ public class Component implements EventHandler<ComponentEvent> {
.addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
new ContainerCompletedTransition())
// Flex while previous flex is still in progress
- .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+ .addTransition(FLEXING, EnumSet.of(FLEXING, STABLE), FLEX,
new FlexComponentTransition())
// container failed while stable
@@ -214,14 +214,12 @@ public class Component implements EventHandler<ComponentEvent> {
new ArrayList<>(component.getAllComponentInstances());
// sort in Most recent -> oldest order, destroy most recent ones.
- Collections.sort(list, Collections.reverseOrder());
+ list.sort(Collections.reverseOrder());
for (int i = 0; i < delta; i++) {
ComponentInstance instance = list.get(i);
// remove the instance
component.compInstances.remove(instance.getCompInstanceName());
component.pendingInstances.remove(instance);
- component.componentMetrics.containersFailed.incr();
- component.componentMetrics.containersRunning.decr();
// decrement id counter
component.instanceIdCounter.decrementAndGet();
instance.destroy();
@@ -282,7 +280,6 @@ public class Component implements EventHandler<ComponentEvent> {
component.compInstanceDispatcher.getEventHandler().handle(
new ComponentInstanceEvent(instance.getContainerId(),
START));
- component.incRunningContainers();
}
}
@@ -294,7 +291,6 @@ public class Component implements EventHandler<ComponentEvent> {
component.compInstanceDispatcher.getEventHandler().handle(
new ComponentInstanceEvent(event.getInstance().getContainerId(),
START));
- component.incRunningContainers();
return checkIfStable(component);
}
}
@@ -413,9 +409,6 @@ public class Component implements EventHandler<ComponentEvent> {
componentMetrics.containersFailed.incr();
scheduler.getServiceMetrics().containersFailed.incr();
- // dec running container
- decRunningContainers();
-
if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
String host = scheduler.getLiveInstances().get(status.getContainerId())
.getNodeId().getHost();
@@ -468,31 +461,33 @@ public class Component implements EventHandler<ComponentEvent> {
}
String ip = instance.getContainerStatus().getIPs().get(0);
String host = instance.getContainerStatus().getHost();
- tokens.put(String.format(COMPONENT_IP,
+ tokens.put(String.format(COMPONENT_INSTANCE_IP,
instance.getCompInstanceName().toUpperCase()), ip);
- tokens.put(String.format(COMPONENT_HOST,
+ tokens.put(String.format(COMPONENT_INSTANCE_HOST,
instance.getCompInstanceName().toUpperCase()), host);
}
}
return tokens;
}
- private void incRunningContainers() {
+ public void incRunningContainers() {
componentMetrics.containersRunning.incr();
scheduler.getServiceMetrics().containersRunning.incr();
}
+ public void decRunningContainers() {
+ componentMetrics.containersRunning.decr();
+ scheduler.getServiceMetrics().containersRunning.decr();
+ }
+
public void incContainersReady() {
componentMetrics.containersReady.incr();
+ scheduler.getServiceMetrics().containersReady.incr();
}
public void decContainersReady() {
componentMetrics.containersReady.decr();
- }
-
- private void decRunningContainers() {
- componentMetrics.containersRunning.decr();
- scheduler.getServiceMetrics().containersRunning.decr();
+ scheduler.getServiceMetrics().containersReady.decr();
}
public int getNumReadyInstances() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 6e585aa..2492454 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -148,7 +148,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
new ContainerStatusRetriever(compInstance.scheduler,
compInstance.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
-
+ compInstance.component.incRunningContainers();
long containerStartTime = System.currentTimeMillis();
try {
ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
@@ -186,6 +186,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
ComponentInstanceEvent event) {
compInstance.component.incContainersReady();
compInstance.containerSpec.setState(ContainerState.READY);
+ if (compInstance.timelineServiceEnabled) {
+ compInstance.serviceTimelinePublisher
+ .componentInstanceBecomeReady(compInstance.containerSpec);
+ }
}
}
@@ -225,6 +229,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.getDiagnostics();
compInstance.diagnostics.append(containerDiag + System.lineSeparator());
+ if (compInstance.getState().equals(READY)) {
+ compInstance.component.decContainersReady();
+ }
+ compInstance.component.decRunningContainers();
boolean shouldExit = false;
// check if it exceeds the failure threshold
if (comp.currentContainerFailure.get() > comp.maxContainerFailurePerComp) {
@@ -250,10 +258,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.submit(compInstance::cleanupRegistry);
if (compInstance.timelineServiceEnabled) {
// record in ATS
- compInstance.serviceTimelinePublisher
- .componentInstanceFinished(compInstance,
- event.getStatus().getExitStatus(), event.getStatus().getState(),
- containerDiag);
+ compInstance.serviceTimelinePublisher.componentInstanceFinished
+ (compInstance, event.getStatus().getExitStatus(), containerDiag);
}
compInstance.containerSpec.setState(ContainerState.STOPPED);
}
@@ -336,7 +342,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
container.setIp(StringUtils.join(",", status.getIPs()));
container.setHostname(status.getHost());
if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceUpdated(container);
+ serviceTimelinePublisher.componentInstanceIPHostUpdated(container);
}
}
updateServiceRecord(yarnRegistryOperations, status);
@@ -410,7 +416,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
- // Release the container , cleanup registry, hdfs dir, and record in ATS
+ // Called when user flexed down the container and ContainerStoppedTransition
+ // is not executed in this case.
+ // Release the container, dec running,
+ // cleanup registry, hdfs dir, and send record to ATS
public void destroy() {
LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
diagnostics.append(getCompInstanceId() + ": Flexed down by user");
@@ -420,10 +429,18 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
.releaseAssignedContainer(container.getId());
getCompSpec().removeContainer(containerSpec);
}
+ // update metrics
+ if (getState() == STARTED) {
+ component.decRunningContainers();
+ }
+ if (getState() == READY) {
+ component.decContainersReady();
+ component.decRunningContainers();
+ }
+
if (timelineServiceEnabled) {
- serviceTimelinePublisher
- .componentInstanceFinished(this, KILLED_BY_APPMASTER, COMPLETE,
- diagnostics.toString());
+ serviceTimelinePublisher.componentInstanceFinished(this,
+ KILLED_BY_APPMASTER, diagnostics.toString());
}
scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 1968e95..a7bd58d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -28,7 +28,7 @@ public class YarnServiceConf {
// Retry settings for container failures
public static final String CONTAINER_RETRY_MAX = "yarn.service.container-failure.retry.max";
- public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval";
+ public static final String CONTAINER_RETRY_INTERVAL = "yarn.service.container-failure.retry-interval-ms";
public static final String AM_RESTART_MAX = "yarn.service.am-restart.max-attempts";
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
@@ -40,6 +40,13 @@ public class YarnServiceConf {
public static final String DEFAULT_API_SERVER_ADDRESS = "0.0.0.0:";
public static final int DEFAULT_API_SERVER_PORT = 9191;
+ public static final String FINAL_LOG_INCLUSION_PATTERN = "yarn.service.log.include-pattern";
+ public static final String FINAL_LOG_EXCLUSION_PATTERN = "yarn.service.log.exclude-pattern";
+
+ public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern";
+ public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern";
+
+
/**
* The yarn service base path:
* Defaults to HomeDir/.yarn/
@@ -98,4 +105,9 @@ public class YarnServiceConf {
Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
return userConf.getPropertyInt(name, systemConf.getInt(name, defaultValue));
}
+
+ public static String get(String name, String defaultVal,
+ Configuration userConf, org.apache.hadoop.conf.Configuration systemConf) {
+ return userConf.getProperty(name, systemConf.get(name, defaultVal));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
index e5ed703..3973759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -87,4 +87,6 @@ public interface YarnServiceConstants {
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
+
+ String CONTENT = "content";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
index 684f655..c4f63ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -70,9 +70,9 @@ public final class MonitorUtils {
try {
switch (readinessCheck.getType()) {
case HTTP:
- return HttpProbe.create(readinessCheck.getProps());
+ return HttpProbe.create(readinessCheck.getProperties());
case PORT:
- return PortProbe.create(readinessCheck.getProps());
+ return PortProbe.create(readinessCheck.getProperties());
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 0d11be2..75c647b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -27,10 +27,13 @@ import org.apache.hadoop.yarn.service.utils.SliderUtils;
import java.io.IOException;
import java.nio.file.Paths;
+import java.text.MessageFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConstants.CONTENT;
+
public abstract class AbstractClientProvider {
public AbstractClientProvider() {
@@ -84,11 +87,15 @@ public abstract class AbstractClientProvider {
throw new IllegalArgumentException("File type is empty");
}
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE)) {
+ if (StringUtils.isEmpty(file.getSrcFile()) &&
+ !file.getProperties().containsKey(CONTENT)) {
+ throw new IllegalArgumentException(MessageFormat.format("For {0} " +
+ "format, either src_file must be specified in ConfigFile," +
+ " or the \"{1}\" key must be specified in " +
+ "the 'properties' field of ConfigFile. ",
+ ConfigFile.TypeEnum.TEMPLATE, CONTENT));
+ }
}
if (!StringUtils.isEmpty(file.getSrcFile())) {
Path p = new Path(file.getSrcFile());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 63fbaae..582c5ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -272,8 +272,8 @@ public class ProviderUtils implements YarnServiceConstants {
break;
}
} else {
- // non-template
- resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+ // If src_file is not specified
+ resolvePropsInConfigFileAndSaveOnHdfs(fs, tokensForSubstitution,
instance, configFile, fileName, remoteFile);
}
}
@@ -297,17 +297,17 @@ public class ProviderUtils implements YarnServiceConstants {
}
}
- private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+ private static void resolvePropsInConfigFileAndSaveOnHdfs(SliderFileSystem fs,
Map<String, String> tokensForSubstitution, ComponentInstance instance,
ConfigFile configFile, String fileName, Path remoteFile)
throws IOException {
// substitute non-template configs
- substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+ substituteMapWithTokens(configFile.getProperties(), tokensForSubstitution);
// write configs onto hdfs
PublishedConfiguration publishedConfiguration =
new PublishedConfiguration(fileName,
- configFile.getProps().entrySet());
+ configFile.getProperties().entrySet());
if (!fs.getFileSystem().exists(remoteFile)) {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(
@@ -343,7 +343,7 @@ public class ProviderUtils implements YarnServiceConstants {
confCopy.set(entry.getKey(), entry.getValue());
}
// substitute properties
- for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+ for (Map.Entry<String, String> entry : configFile.getProperties().entrySet()) {
confCopy.set(entry.getKey(), entry.getValue());
}
// substitute env variables
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
index 7f7f9a1..6c3428a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelineEvent.java
@@ -30,5 +30,7 @@ public enum ServiceTimelineEvent {
COMPONENT_INSTANCE_UNREGISTERED,
- COMPONENT_INSTANCE_UPDATED
+ COMPONENT_INSTANCE_IP_HOST_UPDATE,
+
+ COMPONENT_INSTANCE_BECOME_READY
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index cfe7c1b..c522986 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -20,19 +20,13 @@ package org.apache.hadoop.yarn.service.timelineservice;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.*;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.slf4j.Logger;
@@ -46,6 +40,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import static org.apache.hadoop.yarn.service.api.records.ContainerState.READY;
+import static org.apache.hadoop.yarn.service.api.records.ContainerState.STOPPED;
import static org.apache.hadoop.yarn.service.timelineservice.ServiceTimelineMetricsConstants.DIAGNOSTICS_INFO;
/**
@@ -117,8 +113,8 @@ public class ServiceTimelinePublisher extends CompositeService {
// publish system config - YarnConfiguration
populateTimelineEntity(systemConf.iterator(), service.getId(),
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
- // publish user conf
- publishUserConf(service.getConfiguration(), service.getId(),
+ // publish container conf
+ publishContainerConf(service.getConfiguration(), service.getId(),
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString());
// publish component as separate entity.
@@ -183,7 +179,7 @@ public class ServiceTimelinePublisher extends CompositeService {
}
public void componentInstanceFinished(ComponentInstance instance,
- int exitCode, ContainerState state, String diagnostics) {
+ int exitCode, String diagnostics) {
TimelineEntity entity = createComponentInstanceEntity(
instance.getContainer().getId().toString());
@@ -192,7 +188,7 @@ public class ServiceTimelinePublisher extends CompositeService {
entityInfos.put(ServiceTimelineMetricsConstants.EXIT_STATUS_CODE,
exitCode);
entityInfos.put(DIAGNOSTICS_INFO, diagnostics);
- entityInfos.put(ServiceTimelineMetricsConstants.STATE, state);
+ entityInfos.put(ServiceTimelineMetricsConstants.STATE, STOPPED);
entity.addInfo(entityInfos);
// add an event
@@ -205,7 +201,7 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void componentInstanceUpdated(Container container) {
+ public void componentInstanceIPHostUpdated(Container container) {
TimelineEntity entity = createComponentInstanceEntity(container.getId());
// create info keys
@@ -218,14 +214,27 @@ public class ServiceTimelinePublisher extends CompositeService {
entity.addInfo(entityInfos);
TimelineEvent updateEvent = new TimelineEvent();
- updateEvent
- .setId(ServiceTimelineEvent.COMPONENT_INSTANCE_UPDATED.toString());
+ updateEvent.setId(ServiceTimelineEvent.COMPONENT_INSTANCE_IP_HOST_UPDATE
+ .toString());
updateEvent.setTimestamp(System.currentTimeMillis());
entity.addEvent(updateEvent);
putEntity(entity);
}
+ public void componentInstanceBecomeReady(Container container) {
+ TimelineEntity entity = createComponentInstanceEntity(container.getId());
+ Map<String, Object> entityInfo = new HashMap<>();
+ entityInfo.put(ServiceTimelineMetricsConstants.STATE, READY);
+ entity.addInfo(entityInfo);
+ TimelineEvent updateEvent = new TimelineEvent();
+ updateEvent.setId(ServiceTimelineEvent.COMPONENT_INSTANCE_BECOME_READY
+ .toString());
+ updateEvent.setTimestamp(System.currentTimeMillis());
+ entity.addEvent(updateEvent);
+ putEntity(entity);
+ }
+
private void publishComponents(List<Component> components) {
long currentTimeMillis = System.currentTimeMillis();
for (Component component : components) {
@@ -266,22 +275,19 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
- // publish component specific configurations
- publishUserConf(component.getConfiguration(), component.getName(),
+ // publish container specific configurations
+ publishContainerConf(component.getConfiguration(), component.getName(),
ServiceTimelineEntityType.COMPONENT.toString());
}
}
- private void publishUserConf(Configuration configuration,
+ private void publishContainerConf(Configuration configuration,
String entityId, String entityType) {
- populateTimelineEntity(configuration.getProperties().entrySet().iterator(),
- entityId, entityType);
-
populateTimelineEntity(configuration.getEnv().entrySet().iterator(),
entityId, entityType);
for (ConfigFile configFile : configuration.getFiles()) {
- populateTimelineEntity(configFile.getProps().entrySet().iterator(),
+ populateTimelineEntity(configFile.getProperties().entrySet().iterator(),
entityId, entityType);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
index 8739382..18318aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
@@ -96,9 +96,9 @@ public class TestAppJsonResolve extends Assert {
props.put("k1", "overridden");
props.put("k2", "v2");
files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
+ .PROPERTIES).properties(props));
files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
+ .XML).properties(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(simple.getFiles().get(0)));
assertTrue(files.contains(simple.getFiles().get(1)));
@@ -113,9 +113,9 @@ public class TestAppJsonResolve extends Assert {
props.put("k1", "v1");
files.clear();
files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
- .PROPERTIES).props(props));
+ .PROPERTIES).properties(props));
files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
- .XML).props(Collections.singletonMap("k3", "v3")));
+ .XML).properties(Collections.singletonMap("k3", "v3")));
assertTrue(files.contains(master.getFiles().get(0)));
assertTrue(files.contains(master.getFiles().get(1)));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
index e7c7600..80b4f51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -160,7 +160,7 @@ public class TestServiceTimelinePublisher {
// updated container state
container.setState(ContainerState.READY);
- serviceTimelinePublisher.componentInstanceUpdated(container);
+ serviceTimelinePublisher.componentInstanceIPHostUpdated(container);
lastPublishedEntities =
((DummyTimelineClient) timelineClient).getLastPublishedEntities();
assertEquals(1, lastPublishedEntities.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
index 753a9cd..30fa0eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app-override.json
@@ -10,7 +10,7 @@
{
"type": "PROPERTIES",
"dest_file": "file1",
- "props": {
+ "properties": {
"k1": "v1",
"k2": "v2"
}
@@ -18,7 +18,7 @@
{
"type": "XML",
"dest_file": "file2",
- "props": {
+ "properties": {
"k3": "v3"
}
}
@@ -38,7 +38,7 @@
{
"type": "PROPERTIES",
"dest_file": "file1",
- "props": {
+ "properties": {
"k1": "overridden"
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
new file mode 100644
index 0000000..561df7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -0,0 +1,165 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+This document describes how to configure the services to be deployed on YARN
+
+There are mainly three types of configurations:
+
+* The configurations specific to the custom service running on YARN . E.g. the hbase-site.xml for a Hbase service running on YARN.
+ * It can be specified at both global service level(`Service#configuration`) or component level(`Component#configuration`).
+ * Service-level configs are considered as the default configs for all components and component-level config can override service level config.
+ * All config properties that uses constant variables as described below are subject to substitutions.
+* The configurations specific to YARN service AM. (`Configuration#properties`).
+ * E.g. The `yarn.service.am-restart.max-attempts` which controls how many times
+the framework AM can be retried if it fails. These configs are mainly to control the behavior of the framework AM , rather than the custom services launched by the framework.
+* Some constants such as `SERVICE_NAME` for referring some system properties.
+ * They are substituted by the service AM before writing the config files.
+
+Below describes the details for each type of configurations.
+
+## Configuration for custom service
+Below is how a configuration object typically looks like:
+```
+"configuration" : {
+ "properties" : {
+ "yarn.service.am-restart.max-attempts" : 10 // config for the service AM
+ },
+ "env" : { // The environment variables to be exported when container gets launched
+ "env1" : "val1"
+ },
+ "files" : [ // The list of configuration files to be mounted for the container
+ {
+ "type": "HADOOP_XML", // The format of the config file into which the "properties" are dumped
+ "dest_file": "/etc/hadoop/conf/core-site.xml", // The location where the config file is mounted inside the container
+ "properties" : { // The list of key/value pairs to be dumped into the config file
+ "fs.defaultFS" : "hdfs://myhdfs" // This property will be written into core-site.xml
+ }
+ },
+ {
+ "type": "HADOOP_XML", // The format of the config file.
+ "src_file" : ""hdfs://mycluster/tmp/conf/yarn-site.xml"" // The location of the source config file to be downloaded
+ "dest_file": "/etc/hadoop/conf/yarn-site.xml", // The location where the config file will be mounted inside the container/
+ "properties" : {
+ "yarn.resourcemanager.webapp.address" : "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}" // Merge into (or override existing property in) yarn-site.xml
+ }
+ }
+ ]
+}
+```
+
+* properties: the configurations for service AM. Details below.
+* env : the environment variables to be exported when container gets launched.
+* files : The list of configuration files to be mounted inside the container.
+ * type: The format of the config file(`dest_file`) to be mounted inside the container. If `src_file` is specified, it is also the format for both `src_file` and `dest_file`.
+ * HADOOP_XML : the hadoop xml format. If `src_file` is specified, the file content will be read as parsed in hadoop xml format.
+ * XML : the standard xml format
+ * JSON : the standard JSON format
+ * YAML : the YAML format
+ * PROPERTIES : the java PROPERTIES format
+ * TEMPLATE : the plain text format. If `src_file` is specified, the content of the `src_file` will be written into `dest_file` post constant substitution.
+ If `src_file` is not specified, use `content` as the key in `properties` field, and the value will be the actual content to be written in the `dest_file` post constant substitution. E.g
+
+ ```
+ {
+ "type": "TEMPLATE"
+ "dest_file": "/etc/conf/hello"
+ "properties" : {
+ "content" : "Hello world"
+ }
+ }
+ ```
+ The string `Hello world` will be written into a file located at `/etc/conf/hello` inside the container.
+ * src_file : [optional], the source location of the config file at a network accessible location such as hdfs.
+ * The format of both `src_file` and `dest_file` are defined by `type`.
+ * Currently, It only works with `HADOOP_XML` and `TEMPLATE` type.
+ * The `src_file` will be downloaded by YARN NodeManager and be mounted inside the container as in the location specified by `dest_file`.
+ * If any properties specified in the `properties` field, they are added into (or overwriting existing properties in) the `src_file`.
+ * If `src_file` is not specified, only the properties in the `properties` field will be written into the `dest_file` file.
+ * dest_file : the location where the config file is mounted inside the container. The file format is defined by `type`.
+ dest_file can be an absolute path or a relative path. If it's a relative path, the file will be located in the `$PWD/conf` directory (where `$PWD` is the container local directory which is mounted to all docker containers launched by YARN)
+ * properties : The list of key/value pair configs to be written into the `dest_file` in the format as defined in `type`. If `src_file` is specified, these properties will be added into (or overwriting existing properties in) the `src_file`.
+
+## Configuration for YARN service AM
+This section describes the configurations for configuring the YARN service AM.
+
+These can be specified either in the cluster `yarn-site.xml` at the global level or in the `properties` field of the `Configuration` object as per service basis like below:
+```
+{
+ "configuration" : {
+ "properties" : {
+ "yarn.service.am-restart.max-attempts" : 10
+ }
+ }
+}
+```
+Above config make the service AM to be retried at max 10 times.
+
+#### Available configurations:
+
+| Name | Description |
+| ------------ | ------------- |
+|yarn.service.client-am.retry.max-wait-ms | the max retry time in milliseconds for the service client to talk to the service AM. By default, it is set to 0, which means no retry |
+|yarn.service.client-am.retry-interval-ms | the retry interval in milliseconds for the service client to talk to the service AM. By default, it is 2000, i.e. 2 seconds |
+|yarn.service.container-failure.retry.max | the max number of retries for the container to be auto restarted if it fails. By default, it is set to -1, which means forever.
+|yarn.service.container-failure.retry-interval-ms| the retry interval in milliseconds for the container to be restarted. By default, it is 30000, i.e. 30 seconds |
+|yarn.service.am-restart.max-attempts| the max number of attempts for the framework AM
+|yarn.service.am-resource.memory | the memory size in GB for the framework AM. By default, it is set to 1024
+|yarn.service.queue | the default queue to which the service will be submitted. By default, it is submitted to `default` queue
+|yarn.service.base.path | the root location for the service artifacts on hdfs for a user. By default, it is under ${user_home_dir}/.yarn/
+|yarn.service.container-failure-per-component.threshold | the max number of container failures for a given component before the AM exits.
+|yarn.service.node-blacklist.threshold | Maximum number of container failures on a node before the node is blacklisted by the AM
+|yarn.service.failure-count-reset.window | The interval in seconds when the `yarn.service.container-failure-per-component.threshold` and `yarn.service.node-blacklist.threshold` gets reset. By default, it is 21600, i.e. 6 hours
+|yarn.service.readiness-check-interval.seconds | The interval in seconds between readiness checks. By default, it is 30 seconds
+|yarn.service.log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs after the application completes.
+|yarn.service.log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs after the application completes. If the log file name matches both include and exclude pattern, this file will be excluded.
+|yarn.service.rolling-log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs while app is running.
+|yarn.service.rolling-log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded.
+
+## Constant variables for custom service
+The service framework provides some constant variables for user to configure their services. These variables are either dynamically generated by the system or are static ones such as service name defined by the user.
+User can use these constants in their configurations to be dynamically substituted by the service AM.E.g.
+```
+{
+ "type" : "HADOOP_XML",
+ "dest_file" : "/etc/hadoop/hbase-site.xml",
+ "properties" : {
+ "hbase.regionserver.hostname": "${COMPONENT_INSTANCE_NAME}.${SERVICE_NAME}.${USER}.${DOMAIN}"
+ }
+}
+```
+Here, `COMPONENT_INSTANCE_NAME` and `SERVICE_NAME` are the constants to be substituted by the system.
+
+Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `hwxdev.site`.
+Then, the config will be substituted by the service AM and written in the config file `/etc/hadoop/hbase-site.xml` inside the container as below:
+```
+<property>
+ <name>hbase.regionserver.hostname</name>
+ <value>regionserver-0.hbase.devuser.hwxdev.site</value>
+</property>
+```
+where `regionserver-0` is the actual component instance name assigned by the system for this container.
+
+#### Available constants:
+| Name | Description |
+| ------------ | ------------- |
+| SERVICE_NAME | name of the service defined by the user
+| USER | user who submits the service |
+| DOMAIN | the domain name for the cluster |
+| COMPONENT_NAME | the name for a given component |
+| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |
+| COMPONENT_ID | the monotonically increasing integer for a given component
+| CONTAINER_ID | the YARN container Id for a given container |
+| ${COMPONENT_INSTANCE_NAME}_HOST | the hostname for a component instance (i.e. container), e.g. REGIONSERVER-0_HOST will be substituted by the actual hostname of the component instance. Note all characters must be uppercase. |
+| ${COMPONENT_INSTANCE_NAME}_IP | the ip for a component instance (i.e. container), e.g. REGIONSERVER-0_IP will be substituted by the actual IP address of the component instance. Note all characters must be uppercase. |
+| CLUSTER_FS_URI | the URI of the cluster hdfs |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index e4881ce..0ddd126 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -45,7 +45,7 @@ Note this example requires registry DNS.
{
"type": "ENV",
"dest_file": "/var/www/html/index.html",
- "props": {
+ "properties": {
"content": "<html><header><title>Title</title></header><body>Hello from ${COMPONENT_INSTANCE_NAME}!</body></html>"
}
}
@@ -122,7 +122,7 @@ A readiness check is added for the `httpd` component:
```
"readiness_check": {
"type": "HTTP",
- "props": {
+ "properties": {
"url": "http://${THIS_HOST}:8080"
}
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 58daee5..72c2d3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -52,9 +52,10 @@ The benefits of combining these workloads are two-fold:
* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Descirbes the service discovery mechanism on YARN.
+* [Service Discovery](ServiceDiscovery.md): Describes the service discovery mechanism on YARN.
* [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals.
* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
+* [Configurations](Configurations.md): Describes how to configure the custom services on YARN.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
index 7a2490e..c0e12c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -251,7 +251,7 @@ A config file that needs to be created and made available as a volume in a servi
|type|Config file in the standard format like xml, properties, json, yaml, template.|false|enum (XML, PROPERTIES, JSON, YAML, TEMPLATE, ENV, HADOOP_XML)||
|dest_file|The path that this configuration file should be created as. If it is an absolute path, it will be mounted into the DOCKER container. Absolute paths are only allowed for DOCKER containers. If it is a relative path, only the file name should be provided, and the file will be created in the container local working directory under a folder named conf.|false|string||
|src_file|This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.|false|string||
-|props|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
+|properties|A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file.|false|object||
### Configuration
@@ -260,7 +260,7 @@ Set of configuration properties that can be injected into the service components
|Name|Description|Required|Schema|Default|
|----|----|----|----|----|
-|properties|A blob of key-value pairs of common service properties.|false|object||
+|properties|A blob of key-value pairs for configuring YARN service AM.|false|object||
|env|A blob of key-value pairs which will be appended to the default system properties and handed off to the service at start time. All placeholder references to properties will be substituted before injection.|false|object||
|files|Array of list of files that needs to be created and made available as volumes in the service component containers.|false|ConfigFile array||
@@ -308,7 +308,7 @@ A custom command or a pluggable helper container to determine the readiness of a
|Name|Description|Required|Schema|Default|
|----|----|----|----|----|
|type|E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).|true|enum (HTTP, PORT)||
-|props|A blob of key value pairs that will be used to configure the check.|false|object||
+|properties|A blob of key value pairs that will be used to configure the check.|false|object||
|artifact|Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now. NOT IMPLEMENTED YET|false|Artifact||
@@ -529,14 +529,14 @@ POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -569,14 +569,14 @@ POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
{
"type": "XML",
"dest_file": "/etc/hadoop/conf/core-site.xml",
- "props": {
+ "properties": {
"fs.defaultFS": "${CLUSTER_FS_URI}"
}
},
{
"type": "XML",
"dest_file": "/etc/hbase/conf/hbase-site.xml",
- "props": {
+ "properties": {
"hbase.cluster.distributed": "true",
"hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
"hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index b4bac0d..812f1e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -49,7 +49,7 @@ export default Ember.Controller.extend({
contentPath: 'memory'
}, {
id: 'instances',
- headerTitle: 'Number Of Instances',
+ headerTitle: 'Number of Instances',
contentPath: 'instances',
observePath: true
});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-7332. Compute
effectiveCapacity per each resource vector. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7332. Compute effectiveCapacity per each resource vector. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6df85aaa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6df85aaa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6df85aaa
Branch: refs/heads/YARN-5881
Commit: 6df85aaa28f83445596da0f1068130f841517432
Parents: 60bef62
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Oct 27 10:16:33 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Sat Nov 11 16:25:41 2017 +0530
----------------------------------------------------------------------
.../scheduler/capacity/ParentQueue.java | 66 ++++++++++++--
.../scheduler/capacity/TestParentQueue.java | 94 ++++++++++++++++++++
2 files changed, 153 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6df85aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 75d7641..4426916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
@@ -68,7 +69,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -928,24 +931,25 @@ public class ParentQueue extends AbstractCSQueue {
// Factor to scale down effective resource: When cluster has sufficient
// resources, effective_min_resources will be same as configured
// min_resources.
- float effectiveMinRatio = 1;
+ Resource numeratorForMinRatio = null;
ResourceCalculator rc = this.csContext.getResourceCalculator();
if (getQueueName().equals("root")) {
if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
clusterResource, resourceByLabel, configuredMinResources)) {
- effectiveMinRatio = Resources.divide(rc, clusterResource,
- resourceByLabel, configuredMinResources);
+ numeratorForMinRatio = resourceByLabel;
}
} else {
if (Resources.lessThan(rc, clusterResource,
queueResourceQuotas.getEffectiveMinResource(label),
configuredMinResources)) {
- effectiveMinRatio = Resources.divide(rc, clusterResource,
- queueResourceQuotas.getEffectiveMinResource(label),
- configuredMinResources);
+ numeratorForMinRatio = queueResourceQuotas
+ .getEffectiveMinResource(label);
}
}
+ Map<String, Float> effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
+ configuredMinResources, numeratorForMinRatio);
+
// loop and do this for all child queues
for (CSQueue childQueue : getChildQueues()) {
Resource minResource = childQueue.getQueueResourceQuotas()
@@ -955,7 +959,8 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue.getCapacityConfigType()
.equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
- Resources.multiply(minResource, effectiveMinRatio));
+ getMinResourceNormalized(childQueue.getQueueName(), effectiveMinRatioPerResource,
+ minResource));
// Max resource of a queue should be a minimum of {configuredMaxRes,
// parentMaxRes}. parentMaxRes could be configured value. But if not
@@ -1003,6 +1008,53 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private Resource getMinResourceNormalized(String name, Map<String, Float> effectiveMinRatio,
+ Resource minResource) {
+ Resource ret = Resource.newInstance(minResource);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation nResourceInformation = minResource
+ .getResourceInformation(i);
+
+ Float ratio = effectiveMinRatio.get(nResourceInformation.getName());
+ if (ratio != null) {
+ ret.setResourceValue(i,
+ (long) (nResourceInformation.getValue() * ratio.floatValue()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating min resource for Queue: " + name + " as "
+ + ret.getResourceInformation(i) + ", Actual resource: "
+ + nResourceInformation.getValue() + ", ratio: "
+ + ratio.floatValue());
+ }
+ }
+ }
+ return ret;
+ }
+
+ private Map<String, Float> getEffectiveMinRatioPerResource(
+ Resource configuredMinResources, Resource numeratorForMinRatio) {
+ Map<String, Float> effectiveMinRatioPerResource = new HashMap<>();
+ if (numeratorForMinRatio != null) {
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation nResourceInformation = numeratorForMinRatio
+ .getResourceInformation(i);
+ ResourceInformation dResourceInformation = configuredMinResources
+ .getResourceInformation(i);
+
+ long nValue = nResourceInformation.getValue();
+ long dValue = UnitsConversionUtil.convert(
+ dResourceInformation.getUnits(), nResourceInformation.getUnits(),
+ dResourceInformation.getValue());
+ if (dValue != 0) {
+ effectiveMinRatioPerResource.put(nResourceInformation.getName(),
+ (float) nValue / dValue);
+ }
+ }
+ }
+ return effectiveMinRatioPerResource;
+ }
+
private void deriveCapacityFromAbsoluteConfigurations(String label,
Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6df85aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 25a9774..fe66aba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -68,6 +68,11 @@ import org.mockito.stubbing.Answer;
public class TestParentQueue {
+ private static final Resource QUEUE_B_RESOURCE = Resource
+ .newInstance(14 * 1024, 22);
+ private static final Resource QUEUE_A_RESOURCE = Resource
+ .newInstance(6 * 1024, 10);
+
private static final Log LOG = LogFactory.getLog(TestParentQueue.class);
RMContext rmContext;
@@ -118,6 +123,23 @@ public class TestParentQueue {
LOG.info("Setup top-level queues a and b");
}
+ private void setupSingleLevelQueuesWithAbsoluteResource(
+ CapacitySchedulerConfiguration conf) {
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+ conf.setMinimumResourceRequirement("", Q_A,
+ QUEUE_A_RESOURCE);
+
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ conf.setMinimumResourceRequirement("", Q_B,
+ QUEUE_B_RESOURCE);
+
+ LOG.info("Setup top-level queues a and b with absolute resource");
+ }
+
private FiCaSchedulerApp getMockApplication(int appId, String user) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
doReturn(user).when(application).getUser();
@@ -931,6 +953,78 @@ public class TestParentQueue {
reset(c);
}
+ @Test
+ public void testAbsoluteResourceWithChangeInClusterResource()
+ throws Exception {
+ // Setup queue configs
+ setupSingleLevelQueuesWithAbsoluteResource(csConf);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf,
+ null, CapacitySchedulerConfiguration.ROOT, queues, queues,
+ TestUtils.spyHook);
+
+ // Setup some nodes
+ final int memoryPerNode = 10;
+ int coresPerNode = 16;
+ int numNodes = 2;
+
+ Resource clusterResource = Resources.createResource(
+ numNodes * (memoryPerNode * GB), numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ // Start testing
+ LeafQueue a = (LeafQueue) queues.get(A);
+ LeafQueue b = (LeafQueue) queues.get(B);
+
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE);
+
+ numNodes = 1;
+ clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+ numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ Resource QUEUE_B_RESOURCE_HALF = Resource.newInstance(7 * 1024, 11);
+ Resource QUEUE_A_RESOURCE_HALF = Resource.newInstance(3 * 1024, 5);
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE_HALF);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE_HALF);
+
+ coresPerNode = 40;
+ clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+ numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ Resource QUEUE_B_RESOURCE_70PERC = Resource.newInstance(7 * 1024, 27);
+ Resource QUEUE_A_RESOURCE_30PERC = Resource.newInstance(3 * 1024, 12);
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE_30PERC);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE_70PERC);
+ }
+
@After
public void tearDown() throws Exception {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd
and merge service cmd into application cmd. Contributed by Billie Rinaldi
Posted by su...@apache.org.
YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6
Branch: refs/heads/YARN-5881
Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca
Parents: 8ec9e72
Author: Jian He <ji...@apache.org>
Authored: Sat Oct 28 23:04:19 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:19 2017 -0800
----------------------------------------------------------------------
NOTICE.txt | 4 -
hadoop-project/pom.xml | 6 -
hadoop-yarn-project/hadoop-yarn/bin/yarn | 27 +-
.../hadoop-yarn/conf/yarn-env.sh | 18 +-
.../hadoop-yarn-services-core/pom.xml | 5 -
.../hadoop/yarn/service/ServiceMaster.java | 21 +-
.../hadoop/yarn/service/client/ServiceCLI.java | 113 -------
.../yarn/service/client/ServiceClient.java | 187 +++++------
.../client/params/AbstractActionArgs.java | 139 --------
.../client/params/AbstractArgsDelegate.java | 28 --
.../AbstractClusterBuildingActionArgs.java | 46 ---
.../service/client/params/ActionBuildArgs.java | 39 ---
.../service/client/params/ActionClientArgs.java | 71 -----
.../service/client/params/ActionCreateArgs.java | 46 ---
.../client/params/ActionDependencyArgs.java | 65 ----
.../client/params/ActionDestroyArgs.java | 37 ---
.../service/client/params/ActionExistsArgs.java | 49 ---
.../service/client/params/ActionFlexArgs.java | 50 ---
.../service/client/params/ActionFreezeArgs.java | 56 ----
.../service/client/params/ActionHelpArgs.java | 44 ---
.../service/client/params/ActionKeytabArgs.java | 76 -----
.../service/client/params/ActionListArgs.java | 76 -----
.../client/params/ActionRegistryArgs.java | 218 -------------
.../client/params/ActionResolveArgs.java | 153 ---------
.../client/params/ActionResourceArgs.java | 70 ----
.../service/client/params/ActionStatusArgs.java | 51 ---
.../service/client/params/ActionThawArgs.java | 67 ----
.../service/client/params/ActionTokensArgs.java | 78 -----
.../service/client/params/ActionUpdateArgs.java | 37 ---
.../yarn/service/client/params/ArgOps.java | 156 ---------
.../yarn/service/client/params/Arguments.java | 103 ------
.../yarn/service/client/params/ClientArgs.java | 193 -----------
.../yarn/service/client/params/CommonArgs.java | 245 --------------
.../client/params/ComponentArgsDelegate.java | 52 ---
.../client/params/DontSplitArguments.java | 34 --
.../client/params/LaunchArgsAccessor.java | 30 --
.../client/params/LaunchArgsDelegate.java | 51 ---
.../client/params/OptionArgsDelegate.java | 66 ----
.../client/params/PathArgumentConverter.java | 34 --
.../service/client/params/ServiceAMArgs.java | 50 ---
.../client/params/ServiceAMCreateAction.java | 37 ---
.../service/client/params/SliderActions.java | 79 -----
.../service/client/params/WaitArgsDelegate.java | 42 ---
.../service/client/params/WaitTimeAccessor.java | 24 --
.../hadoop/yarn/service/utils/SliderUtils.java | 6 +-
.../client/TestBuildExternalComponents.java | 19 +-
.../yarn/service/client/TestServiceCLI.java | 125 +++-----
.../hadoop/yarn/client/api/AppAdminClient.java | 222 +++++++++++++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 318 ++++++++++++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++++++---
.../src/site/markdown/YarnCommands.md | 86 ++---
.../src/site/markdown/yarn-service/Examples.md | 12 +-
.../site/markdown/yarn-service/QuickStart.md | 46 ++-
.../markdown/yarn-service/YarnServiceAPI.md | 20 +-
54 files changed, 840 insertions(+), 3245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index f3af2f7..bc7a26f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -582,10 +582,6 @@ Ehcache 3.3.1,
which has the following notices:
* Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
-JCommander (https://github.com/cbeust/jcommander),
-which has the following notices:
- * Copyright 2010 Cedric Beust cedric@beust.com
-
The binary distribution of this product bundles binaries of
snakeyaml (https://bitbucket.org/asomov/snakeyaml),
which has the following notices:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d18c9a8..f99e413 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -146,7 +146,6 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <jcommander.version>1.48</jcommander.version>
</properties>
<dependencyManagement>
@@ -1353,11 +1352,6 @@
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency>
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- <version>${jcommander.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index cc269d8..00596c2 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,8 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
- hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN"
- hadoop_add_subcommand "application" client "prints application(s) report/kill application"
+ hadoop_add_subcommand "app|application" client "prints application(s) report/kill application/manage long running application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
hadoop_add_subcommand "cluster" client "prints cluster information"
@@ -51,7 +50,6 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
- hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
@@ -71,9 +69,10 @@ function yarncmd_case
shift
case ${subcmd} in
- apiserver)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
+ app|application|applicationattempt|container)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+ set -- "${subcmd}" "$@"
+ HADOOP_SUBCMD_ARGS=("$@")
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
@@ -83,11 +82,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
- application|applicationattempt|container)
- HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
- set -- "${subcmd}" "$@"
- HADOOP_SUBCMD_ARGS=("$@")
- ;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
@@ -169,17 +163,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- service)
- HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
- local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
-${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
-${HADOOP_HDFS_HOME}/${HDFS_DIR},\
-${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
-${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
- hadoop_translate_cygwin_path sld
- hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
- ;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 4bd1d3e..d865023 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -150,18 +150,6 @@
#export YARN_ROUTER_OPTS=
###
-# Services API specific parameters
-###
-# Specify the JVM options to be used when starting the services API.
-#
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# See ResourceManager for some examples
-#
-#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
-
-###
# Registry DNS specific parameters
###
# For privileged registry DNS, user to run as after dropping privileges
@@ -172,3 +160,9 @@
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
+
+###
+# YARN Services parameters
+###
+# Directory containing service examples
+# export YARN_SERVICE_EXAMPLES_DIR = $HADOOP_YARN_HOME/share/hadoop/yarn/yarn-service-examples
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 851f73b..6c9a875 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -102,11 +102,6 @@
</build>
<dependencies>
<dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2abdae1..d58e32b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.service;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -34,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@@ -51,7 +52,9 @@ public class ServiceMaster extends CompositeService {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceMaster.class);
- private static ServiceAMArgs amArgs;
+ public static final String YARNFILE_OPTION = "yarnfile";
+
+ private static String serviceDefPath;
protected ServiceContext context;
public ServiceMaster(String name) {
@@ -108,7 +111,7 @@ public class ServiceMaster extends CompositeService {
}
protected Path getAppDir() {
- return new Path(amArgs.getServiceDefPath()).getParent();
+ return new Path(serviceDefPath).getParent();
}
protected ServiceScheduler createServiceScheduler(ServiceContext context)
@@ -119,7 +122,7 @@ public class ServiceMaster extends CompositeService {
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
context.service = ServiceApiUtil
- .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath()));
+ .loadServiceFrom(fs, new Path(serviceDefPath));
LOG.info(context.service.toString());
}
@@ -138,14 +141,18 @@ public class ServiceMaster extends CompositeService {
public static void main(String[] args) throws Exception {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
- amArgs = new ServiceAMArgs(args);
- amArgs.parse();
try {
ServiceMaster serviceMaster = new ServiceMaster("Service Master");
ShutdownHookManager.get()
.addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
YarnConfiguration conf = new YarnConfiguration();
- new GenericOptionsParser(conf, args);
+ Options opts = new Options();
+ opts.addOption(YARNFILE_OPTION, true, "HDFS path to JSON service " +
+ "specification");
+ opts.getOption(YARNFILE_OPTION).setRequired(true);
+ GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);
+ CommandLine cmdLine = parser.getCommandLine();
+ serviceMaster.serviceDefPath = cmdLine.getOptionValue(YARNFILE_OPTION);
serviceMaster.init(conf);
serviceMaster.start();
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
deleted file mode 100644
index 928c06f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import com.beust.jcommander.ParameterException;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
-
-public class ServiceCLI {
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceClient.class);
- protected ServiceClient client;
-
- int exec(ClientArgs args) throws Throwable {
- if (StringUtils.isEmpty(args.getAction())) {
- System.out.println(args.usage());
- return -1;
- }
- switch (args.getAction()) {
- case ACTION_BUILD: // Upload app json onto hdfs
- client.actionBuild(args.getActionBuildArgs());
- break;
- case ACTION_START: // start the app with the pre-uploaded app json on hdfs
- client.actionStart(args.getClusterName());
- break;
- case ACTION_CREATE: // create == build + start
- client.actionCreate(args.getActionCreateArgs());
- break;
- case ACTION_STATUS:
- Service app = client.getStatus(args.getClusterName());
- System.out.println(app);
- break;
- case ACTION_FLEX:
- try {
- client.actionFlexByCLI(args);
- } catch (FileNotFoundException e) {
- System.err.println(
- args.getClusterName() + " doesn't exist: " + e.getMessage());
- return -1;
- }
- break;
- case ACTION_STOP:
- client.actionStop(args.getClusterName(), false);
- break;
- case ACTION_DESTROY: // Destroy can happen only if app is already stopped
- client.actionDestroy(args.getClusterName());
- break;
- case ACTION_DEPENDENCY: // upload dependency jars
- client.actionDependency(args.getActionDependencyArgs());
- break;
- case ACTION_UPDATE:
- client.updateLifetime(args.getClusterName(),
- args.getActionUpdateArgs().lifetime);
- break;
- case ACTION_HELP:
- LOG.info(args.usage());
- break;
- default:
- LOG.info("NOT IMPLEMENTED: " + args.getAction());
- LOG.info(args.usage());
- return -1;
- }
- return 0;
- }
-
- public ServiceCLI() {
- createServiceClient();
- }
-
- protected void createServiceClient() {
- client = new ServiceClient();
- client.init(new YarnConfiguration());
- client.start();
- }
-
- public static void main(String[] args) throws Throwable {
- ClientArgs clientArgs = new ClientArgs(args);
- try {
- clientArgs.parse();
- } catch (ParameterException | SliderException e) {
- System.err.println(e.getMessage());
- System.exit(-1);
- }
- ServiceCLI cli = new ServiceCLI();
- int res = cli.exec(clientArgs);
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index f6d35e4..636b127 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -65,13 +65,6 @@ import org.apache.hadoop.yarn.service.ServiceMaster;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
-import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
@@ -80,7 +73,6 @@ import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
@@ -107,16 +99,14 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class ServiceClient extends CompositeService
- implements SliderExitCodes, YarnServiceConstants {
+public class ServiceClient extends AppAdminClient implements SliderExitCodes,
+ YarnServiceConstants {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
@@ -136,10 +126,6 @@ public class ServiceClient extends CompositeService
private static EnumSet<YarnApplicationState> preRunningStates =
EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
- public ServiceClient() {
- super(ServiceClient.class.getName());
- }
-
@Override protected void serviceInit(Configuration configuration)
throws Exception {
fs = new SliderFileSystem(configuration);
@@ -157,28 +143,60 @@ public class ServiceClient extends CompositeService
super.serviceStop();
}
- private Service loadAppJsonFromLocalFS(
- AbstractClusterBuildingActionArgs args) throws IOException {
- File file = args.getFile();
+ public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
+ Long lifetime, String queue) throws IOException, YarnException {
+ File file = new File(fileName);
+ if (!file.exists() && fileName.equals(file.getName())) {
+ String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
+ String[] examplesDirs;
+ if (examplesDirStr == null) {
+ String yarnHome = System
+ .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
+ examplesDirs = new String[]{
+ yarnHome + "/share/hadoop/yarn/yarn-service-examples",
+ yarnHome + "/yarn-service-examples"
+ };
+ } else {
+ examplesDirs = StringUtils.split(examplesDirStr, ":");
+ }
+ for (String dir : examplesDirs) {
+ file = new File(MessageFormat.format("{0}/{1}/{2}.json",
+ dir, fileName, fileName));
+ if (file.exists()) {
+ break;
+ }
+ // Then look for secondary location.
+ file = new File(MessageFormat.format("{0}/{1}.json",
+ dir, fileName));
+ if (file.exists()) {
+ break;
+ }
+ }
+ }
+ if (!file.exists()) {
+ throw new YarnException("File or example could not be found: " +
+ fileName);
+ }
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
- if (args.lifetime > 0) {
- service.setLifetime(args.lifetime);
+ if (!StringUtils.isEmpty(serviceName)) {
+ service.setName(serviceName);
}
- if (!StringUtils.isEmpty(args.getServiceName())) {
- service.setName(args.getServiceName());
+ if (lifetime != null && lifetime > 0) {
+ service.setLifetime(lifetime);
}
- if (!StringUtils.isEmpty(args.queue)) {
- service.setQueue(args.queue);
+ if (!StringUtils.isEmpty(queue)) {
+ service.setQueue(queue);
}
return service;
}
- public int actionBuild(AbstractClusterBuildingActionArgs args)
- throws IOException, YarnException {
- return actionBuild(loadAppJsonFromLocalFS(args));
+ public int actionSave(String fileName, String serviceName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ return actionBuild(loadAppJsonFromLocalFS(fileName, serviceName,
+ lifetime, queue));
}
public int actionBuild(Service service)
@@ -189,43 +207,13 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionCreate(ActionCreateArgs args)
- throws IOException, YarnException {
- Service serviceDef;
- if (args.file != null) {
- serviceDef = loadAppJsonFromLocalFS(args);
- } else if (!StringUtils.isEmpty(args.example)) {
- // create an example service
- args.file = findExampleService(args);
- serviceDef = loadAppJsonFromLocalFS(args);
- } else {
- throw new YarnException("No service definition provided!");
- }
- actionCreate(serviceDef);
+ public int actionLaunch(String fileName, String serviceName, Long lifetime,
+ String queue) throws IOException, YarnException {
+ actionCreate(loadAppJsonFromLocalFS(fileName, serviceName, lifetime,
+ queue));
return EXIT_SUCCESS;
}
- private File findExampleService(ActionCreateArgs args) throws YarnException {
- String yarnHome = System
- .getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
- // First look for the standard location.
- File file = new File(MessageFormat
- .format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
- yarnHome, args.example, args.example));
- if (file.exists()) {
- return file;
- }
- // Then look for secondary location.
- file = new File(MessageFormat
- .format("{0}/yarn-service-examples/{1}/{2}.json", yarnHome,
- args.example, args.example));
- if (file.exists()) {
- return file;
- }
- throw new YarnException(
- "Example service " + args.example + " does not exist!");
- }
-
public ApplicationId actionCreate(Service service)
throws IOException, YarnException {
String serviceName = service.getName();
@@ -244,14 +232,12 @@ public class ServiceClient extends CompositeService
return appId;
}
- // Called by ServiceCLI
- protected int actionFlexByCLI(ClientArgs args)
- throws YarnException, IOException {
- ActionFlexArgs flexArgs = args.getActionFlexArgs();
+ public int actionFlex(String serviceName, Map<String, String>
+ componentCountStrings) throws YarnException, IOException {
Map<String, Long> componentCounts =
- new HashMap<>(flexArgs.getComponentMap().size());
+ new HashMap<>(componentCountStrings.size());
Service persistedService =
- ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
+ ServiceApiUtil.loadService(fs, serviceName);
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));
@@ -260,8 +246,7 @@ public class ServiceClient extends CompositeService
+ " appId is null, may be not submitted to YARN yet");
}
- for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
- .entrySet()) {
+ for (Map.Entry<String, String> entry : componentCountStrings.entrySet()) {
String compName = entry.getKey();
ServiceApiUtil.validateNameFormat(compName, getConfig());
Component component = persistedService.getComponent(compName);
@@ -272,11 +257,7 @@ public class ServiceClient extends CompositeService
parseNumberOfContainers(component, entry.getValue());
componentCounts.put(compName, numberOfContainers);
}
- // throw usage exception if no changes proposed
- if (componentCounts.size() == 0) {
- actionHelp(ACTION_FLEX, args);
- }
- flexComponents(args.getClusterName(), componentCounts, persistedService);
+ flexComponents(serviceName, componentCounts, persistedService);
return EXIT_SUCCESS;
}
@@ -372,6 +353,11 @@ public class ServiceClient extends CompositeService
return original;
}
+ public int actionStop(String serviceName)
+ throws YarnException, IOException {
+ return actionStop(serviceName, true);
+ }
+
public int actionStop(String serviceName, boolean waitForAppStopped)
throws YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@@ -442,7 +428,8 @@ public class ServiceClient extends CompositeService
return EXIT_SUCCESS;
}
- public int actionDestroy(String serviceName) throws Exception {
+ public int actionDestroy(String serviceName) throws YarnException,
+ IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
verifyNoLiveAppInRM(serviceName, "destroy");
@@ -461,7 +448,11 @@ public class ServiceClient extends CompositeService
throw new YarnException(message);
}
}
- deleteZKNode(serviceName);
+ try {
+ deleteZKNode(serviceName);
+ } catch (Exception e) {
+ throw new IOException("Could not delete zk node for " + serviceName, e);
+ }
String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
try {
getRegistryClient().delete(registryPath, true);
@@ -517,11 +508,6 @@ public class ServiceClient extends CompositeService
return curatorClient;
}
- private int actionHelp(String actionName, CommonArgs args)
- throws YarnException, IOException {
- throw new UsageException(CommonArgs.usage(args, actionName));
- }
-
private void verifyNoLiveAppInRM(String serviceName, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
@@ -634,9 +620,9 @@ public class ServiceClient extends CompositeService
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
- CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
- CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json"));
+ CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
+ serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
@@ -841,6 +827,24 @@ public class ServiceClient extends CompositeService
return ServiceState.ACCEPTED;
}
+ public String getStatusString(String appId)
+ throws IOException, YarnException {
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(ApplicationId.fromString(appId));
+
+ if (appReport.getYarnApplicationState() != RUNNING) {
+ return "";
+ }
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ return "";
+ }
+ ClientAMProtocol amProxy =
+ createAMProxy(appReport.getHost(), appReport.getRpcPort());
+ GetStatusResponseProto response =
+ amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
+ return response.getStatus();
+ }
+
public Service getStatus(String serviceName)
throws IOException, YarnException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@@ -877,7 +881,11 @@ public class ServiceClient extends CompositeService
return this.yarnClient;
}
- public int actionDependency(ActionDependencyArgs args)
+ public int enableFastLaunch() throws IOException, YarnException {
+ return actionDependency(true);
+ }
+
+ public int actionDependency(boolean overwrite)
throws IOException, YarnException {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
@@ -886,11 +894,10 @@ public class ServiceClient extends CompositeService
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
- if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
+ if (fs.isFile(dependencyLibTarGzip) && !overwrite) {
System.out.println(String.format(
- "Dependency libs are already uploaded to %s. Use %s "
- + "if you want to re-upload", dependencyLibTarGzip.toUri(),
- Arguments.ARG_OVERWRITE));
+ "Dependency libs are already uploaded to %s.", dependencyLibTarGzip
+ .toUri()));
return EXIT_SUCCESS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
deleted file mode 100644
index 9b7e2a4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Base args for all actions
- */
-public abstract class AbstractActionArgs extends ArgOps implements Arguments {
- protected static final Logger log =
- LoggerFactory.getLogger(AbstractActionArgs.class);
-
-
- protected AbstractActionArgs() {
- }
-
- /**
- * This is the default parameter
- */
- @Parameter
- public List<String> parameters = new ArrayList<>();
-
- /**
- * get the name: relies on arg 1 being the cluster name in all operations
- * @return the name argument, null if there is none
- */
- public String getServiceName() {
- return (parameters.isEmpty()) ? null : parameters.get(0);
- }
-
- /**
- -D name=value
-
- Define an configuration option which overrides any options in
- the configuration XML files of the image or in the image configuration
- directory. The values will be persisted.
- Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
-
- */
-
- @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true)
- public List<String> definitions = new ArrayList<>();
-
- /**
- * System properties
- */
- @Parameter(names = {ARG_SYSPROP}, arity = 1,
- description = "system properties in the form name value" +
- " These are set after the JVM is started.",
- hidden = true)
- public List<String> sysprops = new ArrayList<>(0);
-
-
- @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true)
- public boolean debug = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 1;
- }
-
- /**
- * Get the name of the action
- * @return the action name
- */
- public abstract String getActionName() ;
-
- /**
- * Get the max #of params expected
- * @return the number of params in the {@link #parameters} field;
- */
- public int getMaxParams() {
- return getMinParams();
- }
-
- public void validate() throws BadCommandArgumentsException, UsageException {
-
- int minArgs = getMinParams();
- int actionArgSize = parameters.size();
- if (minArgs > actionArgSize) {
- throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
- ", Expected minimum " + minArgs + " but got " + actionArgSize);
- }
- int maxArgs = getMaxParams();
- if (maxArgs == -1) {
- maxArgs = minArgs;
- }
- if (actionArgSize > maxArgs) {
- String message = String.format("%s for action %s: limit is %d but saw %d: ",
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- getActionName(), maxArgs,
- actionArgSize);
-
- log.error(message);
- int index = 1;
- StringBuilder buf = new StringBuilder(message);
- for (String actionArg : parameters) {
- log.error("[{}] \"{}\"", index++, actionArg);
- buf.append(" \"").append(actionArg).append("\" ");
- }
- throw new BadCommandArgumentsException(buf.toString());
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + ": " + getActionName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
deleted file mode 100644
index 457e357..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.client.params.ArgOps;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-
-/**
- * Base class for all the delegates
- */
-public class AbstractArgsDelegate extends ArgOps implements Arguments {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
deleted file mode 100644
index 4ecbe9c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-
-import java.io.File;
-
-/**
- * Abstract Action to build things; shares args across build and
- * list
- */
-public abstract class AbstractClusterBuildingActionArgs
- extends AbstractActionArgs {
- @Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
- description = "The path to the service definition file in JSON format.")
- public File file;
-
- public File getFile() {
- return file;
- }
-
- @Parameter(names = {
- ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
- public String queue;
-
- @Parameter(names = {
- ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
- public long lifetime;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
deleted file mode 100644
index 0228125..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-@Parameters(commandNames = { SliderActions.ACTION_BUILD},
- commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
-
-public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_BUILD;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException {
- if (file == null) {
- throw new BadCommandArgumentsException("No service definition provided.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
deleted file mode 100644
index c43d61a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
- commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
-
-public class ActionClientArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CLIENT;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install client")
- public boolean install;
-
- @Parameter(names = {ARG_NAME},
- description = "The name of the service")
- public String name;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package")
- public String packageURI;
-
- @Parameter(names = {ARG_DEST},
- description = "The location where to install the client")
- public File installLocation;
-
- @Parameter(names = {ARG_CONFIG},
- description = "Client configuration")
- public File clientConfig;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
deleted file mode 100644
index 039f528..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-@Parameters(commandNames = { SliderActions.ACTION_CREATE},
- commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
-
-public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
-
- @Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
- description = "The name of the example service such as sleeper")
- public String example;
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_CREATE;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException {
- if (file == null && example == null) {
- throw new BadCommandArgumentsException("No service definition provided.");
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
deleted file mode 100644
index 699e6ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client.params;
-
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
- commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
-public class ActionDependencyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DEPENDENCY;
- }
-
- @Parameter(names = { ARG_UPLOAD }, required = true,
- description = "Upload AM libraries to HDFS for this client version")
- public boolean upload;
-
- @Parameter(names = { ARG_OVERWRITE },
- description = "Overwrite current uploaded dependency libs")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- *
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- if (!upload) {
- throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
deleted file mode 100644
index 8c41c04..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
- commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
-
-public class ActionDestroyArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_DESTROY;
- }
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
deleted file mode 100644
index 088ad47..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-import java.io.File;
-
-@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
- commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
-
-public class ActionExistsArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_EXISTS;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "verify that the service is running")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "verify that the service is in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "output file for any service report")
- public File out;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
deleted file mode 100644
index b7acf58..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-
-import java.util.List;
-import java.util.Map;
-
-@Parameters(commandNames = { SliderActions.ACTION_FLEX},
- commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
-
-public class ActionFlexArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_FLEX;
- }
-
- @ParametersDelegate
- public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
-
- /**
- * Get the component mapping (may be empty, but never null)
- * @return mapping
- * @throws BadCommandArgumentsException parse problem
- */
- public Map<String, String> getComponentMap() throws
- BadCommandArgumentsException {
- return componentDelegate.getComponentMap();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
deleted file mode 100644
index aecf0eb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-@Parameters(commandNames = { SliderActions.ACTION_STOP },
- commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
-
-public class ActionFreezeArgs extends AbstractActionArgs implements
- WaitTimeAccessor {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_STOP;
- }
-
- public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
- @ParametersDelegate
- public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
-
- @Override
- public int getWaittime() {
- return waitDelegate.getWaittime();
- }
-
- @Override
- public void setWaittime(int waittime) {
- waitDelegate.setWaittime(waittime);
- }
-
- @Parameter(names={ARG_MESSAGE},
- description = "reason for the operation")
- public String message = FREEZE_COMMAND_ISSUED;
-
- @Parameter(names = {ARG_FORCE},
- description = "force the operation")
- public boolean force;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
deleted file mode 100644
index 51aa88a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-/**
- * The Help command
- */
-@Parameters(commandNames = { SliderActions.ACTION_HELP},
- commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
-public class ActionHelpArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_HELP;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
deleted file mode 100644
index 7e51457..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
-
-public class ActionKeytabArgs extends AbstractActionArgs {
-
- public ActionKeytabArgs() {
- super();
- }
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTABINSTALL},
- description = "Install the keytab")
- public boolean install;
-
- @Parameter(names = {ARG_KEYTABDELETE},
- description = "Delete the keytab")
- public boolean delete;
-
- @Parameter(names = {ARG_KEYTABLIST},
- description = "List of installed keytabs")
- public boolean list;
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path or name of the keytab")
- public String keytab;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
deleted file mode 100644
index c05e602..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_LIST;
- }
-
- @Parameter(names = {ARG_LIVE},
- description = "List only live service instances")
- public boolean live;
-
- @Parameter(names = {ARG_STATE},
- description = "list only applications in the specific YARN state")
- public String state = "";
-
- @Parameter(names = {ARG_VERBOSE},
- description = "print out information in details")
- public boolean verbose = false;
-
- @Parameter(names = {ARG_CONTAINERS},
- description = "List containers of a service instance")
- public boolean containers;
-
- @Parameter(names = {ARG_VERSION},
- description = "Filter containers by app version (used with " +
- ARG_CONTAINERS + ")")
- public String version;
-
- @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
- description = "Filter containers by component names (used with " +
- ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index 3e53418..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- *
- * --instance {app name}, if a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
- commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
- public static final String USAGE =
- "Usage: " + SliderActions.ACTION_REGISTRY
- + " ("
- + Arguments.ARG_LIST + "|"
- + Arguments.ARG_LISTCONF + "|"
- + Arguments.ARG_LISTEXP + "|"
- + Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "|"
- + Arguments.ARG_GETEXP + "> "
- + Arguments.ARG_NAME + " <name> "
- + " )"
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_USER + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
- + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
- + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
- + System.getProperty("line.separator")
- + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
- ;
- public ActionRegistryArgs() {
- }
-
- public ActionRegistryArgs(String name) {
- this.name = name;
- }
-
- @Override
- public String getActionName() {
- return ACTION_REGISTRY;
- }
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- @Override
- public int getMinParams() {
- return 0;
- }
-
- @Parameter(names = {ARG_LIST},
- description = "list services")
- public boolean list;
-
- @Parameter(names = {ARG_LISTCONF},
- description = "list configurations")
- public boolean listConf;
-
- @Parameter(names = {ARG_GETCONF},
- description = "get configuration")
- public String getConf;
-
- @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
- public boolean listExports;
-
- @Parameter(names = {ARG_GETEXP},
- description = "get export")
- public String getExport;
-
- @Parameter(names = {ARG_LISTFILES},
- description = "list files")
- public String listFiles;
-
- @Parameter(names = {ARG_GETFILES},
- description = "get files")
- public String getFiles;
-
- //--format
- @Parameter(names = ARG_FORMAT,
- description = "Format for a response: <xml|json|properties>")
- public String format = ConfigFormat.XML.toString() ;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
- description = "Output destination")
- public File out;
-
- @Parameter(names = {ARG_NAME},
- description = "name of an instance")
- public String name;
-
- @Parameter(names = {ARG_SERVICETYPE},
- description = "optional service type")
- public String serviceType = YarnServiceConstants.APP_TYPE;
-
- @Parameter(names = {ARG_VERBOSE},
- description = "verbose output")
- public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
- @Parameter(names = {ARG_USER},
- description = "the name of the user whose service is being resolved")
- public String user;
-
- /**
- * validate health of all the different operations
- * @throws BadCommandArgumentsException
- */
- @Override
- public void validate() throws BadCommandArgumentsException, UsageException {
- super.validate();
-
- //verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles) + s(getExport);
- int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
- int set = lists + gets;
- if (set > 1) {
- throw new UsageException(USAGE);
- }
-
- if (out != null && ( set == 0)) {
- throw new UsageException("output path"
- + " is only supported on 'get' operations: ");
- }
- if (!list && !is(name)) {
- throw new UsageException("Argument " + ARG_NAME
- +" missing: ");
-
- }
- }
-
- private int s(String arg) {
- return is(arg) ? 1 : 0;
- }
-
- private boolean is(String arg) {
- return arg != null;
- }
-
- private int s(boolean arg) {
- return arg ? 1 : 0;
- }
-
- private String ifdef(String arg, boolean val) {
- return val ? (arg + " "): "";
- }
-
- private String ifdef(String arg, String val) {
- if (is(val)) {
- return arg + " " + val + " ";
- } else {
- return "";
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder(ACTION_REGISTRY);
- sb.append(' ');
- sb.append(ifdef(ARG_LIST, list));
- sb.append(ifdef(ARG_LISTCONF, listConf));
- sb.append(ifdef(ARG_LISTFILES, listFiles));
- sb.append(ifdef(ARG_GETCONF, getConf));
- sb.append(ifdef(ARG_GETFILES, getFiles));
-
- sb.append(ifdef(ARG_NAME, name));
- sb.append(ifdef(ARG_SERVICETYPE, serviceType));
-
-
- sb.append(ifdef(ARG_VERBOSE, verbose));
- sb.append(ifdef(ARG_INTERNAL, internal));
-
- if (out != null) {
- sb.append(ifdef(ARG_OUTPUT, out.toString()));
- }
- sb.append(ifdef(ARG_FORMAT, format));
-
- return sb.toString();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-7388. TestAMRestart should be
scheduler agnostic.
Posted by su...@apache.org.
YARN-7388. TestAMRestart should be scheduler agnostic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1382a18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1382a18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1382a18
Branch: refs/heads/YARN-5881
Commit: a1382a18dff8a70aa25240d6fbba6e22832a7679
Parents: 6c32dda
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Nov 9 10:49:50 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Nov 9 10:49:50 2017 -0800
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 8 +++++
.../scheduler/SchedulerUtils.java | 13 +++++++
.../scheduler/capacity/CapacityScheduler.java | 6 ++++
.../scheduler/fair/FairScheduler.java | 10 ++++++
.../scheduler/fifo/FifoScheduler.java | 10 ++++++
.../applicationsmanager/TestAMRestart.java | 36 +++++++-------------
6 files changed, 59 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 53f43e4..7308fd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -1362,6 +1362,14 @@ public abstract class AbstractYarnScheduler
}
/**
+ * Kill a RMContainer. This is meant to be called in tests only to simulate
+ * AM container failures.
+ * @param container the container to kill
+ */
+ @VisibleForTesting
+ public abstract void killContainer(RMContainer container);
+
+ /**
* Update internal state of the scheduler. This can be useful for scheduler
* implementations that maintain some state that needs to be periodically
* updated; for example, metrics or queue resources. It will be called by the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c558b8d..32f5824 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -101,6 +101,19 @@ public class SchedulerUtils {
ContainerExitStatus.ABORTED, diagnostics);
}
+
+ /**
+ * Utility to create a {@link ContainerStatus} for killed containers.
+ * @param containerId {@link ContainerId} of the killed container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for a killed container
+ */
+ public static ContainerStatus createKilledContainerStatus(
+ ContainerId containerId, String diagnostics) {
+ return createAbnormalContainerStatus(containerId,
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, diagnostics);
+ }
+
/**
* Utility to create a {@link ContainerStatus} during exceptional
* circumstances.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index db69042..5e172b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1759,6 +1759,12 @@ public class CapacityScheduler extends
}
}
+ @VisibleForTesting
+ @Override
+ public void killContainer(RMContainer container) {
+ markContainerForKillable(container);
+ }
+
public void markContainerForKillable(
RMContainer killableContainer) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 8ea07ab..7f1b91e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -786,6 +786,16 @@ public class FairScheduler extends
incrAllocation);
}
+ @VisibleForTesting
+ @Override
+ public void killContainer(RMContainer container) {
+ ContainerStatus status = SchedulerUtils.createKilledContainerStatus(
+ container.getContainerId(),
+ "Killed by RM to simulate an AM container failure");
+ LOG.info("Killing container " + container);
+ completedContainer(container, status, RMContainerEventType.KILL);
+ }
+
@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask, List<ContainerId> release,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 185d426..3288912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -975,6 +975,16 @@ public class FifoScheduler extends
updateAvailableResourcesMetrics();
}
+ @VisibleForTesting
+ @Override
+ public void killContainer(RMContainer container) {
+ ContainerStatus status = SchedulerUtils.createKilledContainerStatus(
+ container.getContainerId(),
+ "Killed by RM to simulate an AM container failure");
+ LOG.info("Killing container " + container);
+ completedContainer(container, status, RMContainerEventType.KILL);
+ }
+
@Override
public synchronized void recoverContainersOnNode(
List<NMContainerStatus> containerReports, RMNode nm) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1382a18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index 9d0d879..c43069b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -51,10 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
@@ -376,8 +374,6 @@ public class TestAMRestart {
@Test(timeout = 100000)
public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
@@ -389,12 +385,12 @@ public class TestAMRestart {
RMApp app1 = rm1.submitApp(200);
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- CapacityScheduler scheduler =
- (CapacityScheduler) rm1.getResourceScheduler();
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
ContainerId amContainer =
ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
// Preempt the next attempt;
- scheduler.markContainerForKillable(scheduler.getRMContainer(amContainer));
+ scheduler.killContainer(scheduler.getRMContainer(amContainer));
rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(scheduler,
@@ -414,7 +410,7 @@ public class TestAMRestart {
// Preempt the second attempt.
ContainerId amContainer2 =
ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
- scheduler.markContainerForKillable(scheduler.getRMContainer(amContainer2));
+ scheduler.killContainer(scheduler.getRMContainer(amContainer2));
rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(scheduler,
@@ -503,8 +499,6 @@ public class TestAMRestart {
@Test(timeout = 100000)
public void testMaxAttemptOneMeansOne() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
@@ -516,12 +510,12 @@ public class TestAMRestart {
RMApp app1 = rm1.submitApp(200);
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- CapacityScheduler scheduler =
- (CapacityScheduler) rm1.getResourceScheduler();
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
ContainerId amContainer =
ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
// Preempt the attempt;
- scheduler.markContainerForKillable(scheduler.getRMContainer(amContainer));
+ scheduler.killContainer(scheduler.getRMContainer(amContainer));
rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
TestSchedulerUtils.waitSchedulerApplicationAttemptStopped(scheduler,
@@ -539,8 +533,6 @@ public class TestAMRestart {
@Test(timeout = 60000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
@@ -556,8 +548,8 @@ public class TestAMRestart {
RMApp app1 = rm1.submitApp(200);
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- CapacityScheduler scheduler =
- (CapacityScheduler) rm1.getResourceScheduler();
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
ContainerId amContainer =
ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
@@ -577,7 +569,7 @@ public class TestAMRestart {
// Forcibly preempt the am container;
amContainer = ContainerId.newContainerId(am2.getApplicationAttemptId(), 1);
- scheduler.markContainerForKillable(scheduler.getRMContainer(amContainer));
+ scheduler.killContainer(scheduler.getRMContainer(amContainer));
rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
Assert.assertFalse(attempt2.shouldCountTowardsMaxAttemptRetry());
@@ -619,8 +611,6 @@ public class TestAMRestart {
public void testRMRestartOrFailoverNotCountedForAMFailures()
throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
@@ -631,8 +621,8 @@ public class TestAMRestart {
MockRM rm1 = new MockRM(conf);
MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
rm1.start();
- CapacityScheduler scheduler =
- (CapacityScheduler) rm1.getResourceScheduler();
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
nm1.registerNode();
@@ -694,8 +684,6 @@ public class TestAMRestart {
@Test (timeout = 120000)
public void testRMAppAttemptFailuresValidityInterval() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HADOOP-8522.
ResetableGzipOutputStream creates invalid gzip files when finish() and
resetState() are used. Contributed by Mike Percy
Posted by su...@apache.org.
HADOOP-8522. ResetableGzipOutputStream creates invalid gzip files when finish() and resetState() are used. Contributed by Mike Percy
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/796a0d3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/796a0d3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/796a0d3a
Branch: refs/heads/YARN-5881
Commit: 796a0d3a5c661f0c3b23af9c0db2d8f3db83c322
Parents: 6d201f7
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Nov 10 16:29:36 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Nov 10 17:41:29 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/compress/GzipCodec.java | 37 +++-
.../hadoop/io/compress/TestGzipCodec.java | 169 +++++++++++++++++++
2 files changed, 201 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/796a0d3a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
index 11fcf60..9bd861d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
@@ -41,27 +41,54 @@ import org.apache.hadoop.io.compress.zlib.ZlibFactory;
@InterfaceStability.Evolving
public class GzipCodec extends DefaultCodec {
/**
- * A bridge that wraps around a DeflaterOutputStream to make it
+ * A bridge that wraps around a DeflaterOutputStream to make it
* a CompressionOutputStream.
*/
@InterfaceStability.Evolving
protected static class GzipOutputStream extends CompressorStream {
private static class ResetableGZIPOutputStream extends GZIPOutputStream {
+ /**
+ * Fixed ten-byte gzip header. See {@link GZIPOutputStream}'s source for
+ * details.
+ */
+ private static final byte[] GZIP_HEADER = new byte[] {
+ 0x1f, (byte) 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+ private boolean reset = false;
public ResetableGZIPOutputStream(OutputStream out) throws IOException {
super(out);
}
- public void resetState() throws IOException {
- def.reset();
+ public synchronized void resetState() throws IOException {
+ reset = true;
+ }
+
+ @Override
+ public synchronized void write(byte[] buf, int off, int len)
+ throws IOException {
+ if (reset) {
+ def.reset();
+ crc.reset();
+ out.write(GZIP_HEADER);
+ reset = false;
+ }
+ super.write(buf, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ reset = false;
+ super.close();
}
+
}
public GzipOutputStream(OutputStream out) throws IOException {
super(new ResetableGZIPOutputStream(out));
}
-
+
/**
* Allow children types to put a different type in here.
* @param out the Deflater stream to use
@@ -69,7 +96,7 @@ public class GzipCodec extends DefaultCodec {
protected GzipOutputStream(CompressorStream out) {
super(out);
}
-
+
@Override
public void close() throws IOException {
out.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/796a0d3a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java
new file mode 100644
index 0000000..c8c1a47
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.compress;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Verify resettable compressor.
+ */
+public class TestGzipCodec {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestGzipCodec.class);
+
+ private static final String DATA1 = "Dogs don't know it's not bacon!\n";
+ private static final String DATA2 = "It's baconnnn!!\n";
+ private GzipCodec codec = new GzipCodec();
+
+ @Before
+ public void setUp() {
+ codec.setConf(new Configuration(false));
+ }
+
+ // Test simple compression.
+ @Test
+ public void testSingleCompress() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ CompressionOutputStream cmpOut = codec.createOutputStream(baos);
+ cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+ cmpOut.finish();
+ cmpOut.close();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ GZIPInputStream cmpIn = new GZIPInputStream(bais);
+ byte[] buf = new byte[1024];
+ int len = cmpIn.read(buf);
+ String result = new String(buf, 0, len, StandardCharsets.UTF_8);
+ assertEquals("Input must match output", DATA1, result);
+ }
+
+ // Test multi-member gzip file created via finish(), resetState().
+ @Test
+ public void testResetCompress() throws IOException {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+ cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+ cmpOut.finish();
+ cmpOut.resetState();
+ cmpOut.write(DATA2.getBytes(StandardCharsets.UTF_8));
+ cmpOut.finish();
+ cmpOut.close();
+ dob.close();
+
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(dob.getData(), 0, dob.getLength());
+ CompressionInputStream cmpIn = codec.createInputStream(dib);
+ byte[] buf = new byte[1024];
+ StringBuilder result = new StringBuilder();
+ int len = 0;
+ while (true) {
+ len = cmpIn.read(buf);
+ if (len < 0) {
+ break;
+ }
+ result.append(new String(buf, 0, len, StandardCharsets.UTF_8));
+ }
+ assertEquals("Output must match input", DATA1 + DATA2, result.toString());
+ }
+
+ // ensure all necessary methods are overwritten
+ @Test
+ public void testWriteOverride() throws IOException {
+ Random r = new Random();
+ long seed = r.nextLong();
+ LOG.info("seed: " + seed);
+ r.setSeed(seed);
+ byte[] buf = new byte[128];
+ r.nextBytes(buf);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+ cmpOut.write(buf);
+ int i = r.nextInt(128 - 10);
+ int l = r.nextInt(128 - i);
+ cmpOut.write(buf, i, l);
+ cmpOut.write((byte)(r.nextInt() & 0xFF));
+ cmpOut.close();
+
+ r.setSeed(seed);
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(dob.getData(), 0, dob.getLength());
+ CompressionInputStream cmpIn = codec.createInputStream(dib);
+ byte[] vbuf = new byte[128];
+ assertEquals(128, cmpIn.read(vbuf));
+ assertArrayEquals(buf, vbuf);
+ r.nextBytes(vbuf);
+ int vi = r.nextInt(128 - 10);
+ int vl = r.nextInt(128 - vi);
+ assertEquals(vl, cmpIn.read(vbuf, 0, vl));
+ assertArrayEquals(Arrays.copyOfRange(buf, i, i + l),
+ Arrays.copyOf(vbuf, vl));
+ assertEquals(r.nextInt() & 0xFF, cmpIn.read());
+ assertEquals(-1, cmpIn.read());
+ }
+
+ // don't write a new header if no data are written after reset
+ @Test
+ public void testIdempotentResetState() throws IOException {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ CompressionOutputStream cmpOut = codec.createOutputStream(dob);
+ cmpOut.write(DATA1.getBytes(StandardCharsets.UTF_8));
+ cmpOut.finish();
+ cmpOut.finish();
+ cmpOut.finish();
+ cmpOut.resetState();
+ cmpOut.resetState();
+ cmpOut.finish();
+ cmpOut.resetState();
+ cmpOut.close();
+ dob.close();
+
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(dob.getData(), 0, dob.getLength());
+ CompressionInputStream cmpIn = codec.createInputStream(dib);
+ byte[] buf = new byte[1024];
+ StringBuilder result = new StringBuilder();
+ int len = 0;
+ while (true) {
+ len = cmpIn.read(buf);
+ if (len < 0) {
+ break;
+ }
+ result.append(new String(buf, 0, len, StandardCharsets.UTF_8));
+ }
+ assertEquals("Output must match input", DATA1, result.toString());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-7437. Rename PlacementSet and
SchedulingPlacementSet. (Wangda Tan via kkaranasos)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
new file mode 100644
index 0000000..7f89435
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -0,0 +1,422 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import org.apache.commons.collections.IteratorUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is an implementation of the {@link AppPlacementAllocator} that takes
+ * into account locality preferences (node, rack, any) when allocating
+ * containers.
+ */
+public class LocalityAppPlacementAllocator<N extends SchedulerNode>
+ implements AppPlacementAllocator<N> {
+ private static final Log LOG =
+ LogFactory.getLog(LocalityAppPlacementAllocator.class);
+
+ private final Map<String, ResourceRequest> resourceRequestMap =
+ new ConcurrentHashMap<>();
+ private AppSchedulingInfo appSchedulingInfo;
+ private volatile String primaryRequestedPartition =
+ RMNodeLabelsManager.NO_LABEL;
+
+ private final ReentrantReadWriteLock.ReadLock readLock;
+ private final ReentrantReadWriteLock.WriteLock writeLock;
+
+ public LocalityAppPlacementAllocator(AppSchedulingInfo info) {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ this.appSchedulingInfo = info;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Iterator<N> getPreferredNodeIterator(
+ CandidateNodeSet<N> candidateNodeSet) {
+ // Now only handle the case that single node in the candidateNodeSet
+ // TODO, Add support to multi-hosts inside candidateNodeSet which is passed
+ // in.
+
+ N singleNode = CandidateNodeSetUtils.getSingleNode(candidateNodeSet);
+ if (null != singleNode) {
+ return IteratorUtils.singletonIterator(singleNode);
+ }
+
+ return IteratorUtils.emptyIterator();
+ }
+
+ private boolean hasRequestLabelChanged(ResourceRequest requestOne,
+ ResourceRequest requestTwo) {
+ String requestOneLabelExp = requestOne.getNodeLabelExpression();
+ String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
+ // First request label expression can be null and second request
+ // is not null then we have to consider it as changed.
+ if ((null == requestOneLabelExp) && (null != requestTwoLabelExp)) {
+ return true;
+ }
+ // If the label is not matching between both request when
+ // requestOneLabelExp is not null.
+ return ((null != requestOneLabelExp) && !(requestOneLabelExp
+ .equals(requestTwoLabelExp)));
+ }
+
+ private void updateNodeLabels(ResourceRequest request) {
+ String resourceName = request.getResourceName();
+ if (resourceName.equals(ResourceRequest.ANY)) {
+ ResourceRequest previousAnyRequest =
+ getResourceRequest(resourceName);
+
+ // When there is change in ANY request label expression, we should
+ // update label for all resource requests already added of same
+ // priority as ANY resource request.
+ if ((null == previousAnyRequest) || hasRequestLabelChanged(
+ previousAnyRequest, request)) {
+ for (ResourceRequest r : resourceRequestMap.values()) {
+ if (!r.getResourceName().equals(ResourceRequest.ANY)) {
+ r.setNodeLabelExpression(request.getNodeLabelExpression());
+ }
+ }
+ }
+ } else{
+ ResourceRequest anyRequest = getResourceRequest(ResourceRequest.ANY);
+ if (anyRequest != null) {
+ request.setNodeLabelExpression(anyRequest.getNodeLabelExpression());
+ }
+ }
+ }
+
+ @Override
+ public ResourceRequestUpdateResult updateResourceRequests(
+ Collection<ResourceRequest> requests,
+ boolean recoverPreemptedRequestForAContainer) {
+ try {
+ this.writeLock.lock();
+
+ ResourceRequestUpdateResult updateResult = null;
+
+ // Update resource requests
+ for (ResourceRequest request : requests) {
+ String resourceName = request.getResourceName();
+
+ // Update node labels if required
+ updateNodeLabels(request);
+
+ // Increment number of containers if recovering preempted resources
+ ResourceRequest lastRequest = resourceRequestMap.get(resourceName);
+ if (recoverPreemptedRequestForAContainer && lastRequest != null) {
+ request.setNumContainers(lastRequest.getNumContainers() + 1);
+ }
+
+ // Update asks
+ resourceRequestMap.put(resourceName, request);
+
+ if (resourceName.equals(ResourceRequest.ANY)) {
+ String partition = request.getNodeLabelExpression() == null ?
+ RMNodeLabelsManager.NO_LABEL :
+ request.getNodeLabelExpression();
+
+ this.primaryRequestedPartition = partition;
+
+ //update the applications requested labels set
+ appSchedulingInfo.addRequestedPartition(partition);
+
+ updateResult = new ResourceRequestUpdateResult(lastRequest, request);
+ }
+ }
+ return updateResult;
+ } finally {
+ this.writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Map<String, ResourceRequest> getResourceRequests() {
+ return resourceRequestMap;
+ }
+
+ private ResourceRequest getResourceRequest(String resourceName) {
+ return resourceRequestMap.get(resourceName);
+ }
+
+ @Override
+ public PendingAsk getPendingAsk(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ if (null == request) {
+ return PendingAsk.ZERO;
+ } else{
+ return new PendingAsk(request.getCapability(),
+ request.getNumContainers());
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ @Override
+ public int getOutstandingAsksCount(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ if (null == request) {
+ return 0;
+ } else{
+ return request.getNumContainers();
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey,
+ ResourceRequest offSwitchRequest) {
+ int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
+ offSwitchRequest.setNumContainers(numOffSwitchContainers);
+
+ // Do we have any outstanding requests?
+ // If there is nothing, we need to deactivate this application
+ if (numOffSwitchContainers == 0) {
+ appSchedulingInfo.getSchedulerKeys().remove(schedulerRequestKey);
+ appSchedulingInfo.checkForDeactivation();
+ resourceRequestMap.remove(ResourceRequest.ANY);
+ if (resourceRequestMap.isEmpty()) {
+ appSchedulingInfo.removeAppPlacement(schedulerRequestKey);
+ }
+ }
+
+ appSchedulingInfo.decPendingResource(
+ offSwitchRequest.getNodeLabelExpression(),
+ offSwitchRequest.getCapability());
+ }
+
+ public ResourceRequest cloneResourceRequest(ResourceRequest request) {
+ ResourceRequest newRequest = ResourceRequest.newBuilder()
+ .priority(request.getPriority())
+ .allocationRequestId(request.getAllocationRequestId())
+ .resourceName(request.getResourceName())
+ .capability(request.getCapability())
+ .numContainers(1)
+ .relaxLocality(request.getRelaxLocality())
+ .nodeLabelExpression(request.getNodeLabelExpression()).build();
+ return newRequest;
+ }
+
+ /**
+ * The {@link ResourceScheduler} is allocating data-local resources to the
+ * application.
+ */
+ private void allocateRackLocal(SchedulerRequestKey schedulerKey,
+ SchedulerNode node, ResourceRequest rackLocalRequest,
+ List<ResourceRequest> resourceRequests) {
+ // Update future requirements
+ decResourceRequest(node.getRackName(), rackLocalRequest);
+
+ ResourceRequest offRackRequest = resourceRequestMap.get(
+ ResourceRequest.ANY);
+ decrementOutstanding(schedulerKey, offRackRequest);
+
+ // Update cloned RackLocal and OffRack requests for recovery
+ resourceRequests.add(cloneResourceRequest(rackLocalRequest));
+ resourceRequests.add(cloneResourceRequest(offRackRequest));
+ }
+
+ /**
+ * The {@link ResourceScheduler} is allocating data-local resources to the
+ * application.
+ */
+ private void allocateOffSwitch(SchedulerRequestKey schedulerKey,
+ ResourceRequest offSwitchRequest,
+ List<ResourceRequest> resourceRequests) {
+ // Update future requirements
+ decrementOutstanding(schedulerKey, offSwitchRequest);
+ // Update cloned OffRack requests for recovery
+ resourceRequests.add(cloneResourceRequest(offSwitchRequest));
+ }
+
+
+ /**
+ * The {@link ResourceScheduler} is allocating data-local resources to the
+ * application.
+ */
+ private void allocateNodeLocal(SchedulerRequestKey schedulerKey,
+ SchedulerNode node, ResourceRequest nodeLocalRequest,
+ List<ResourceRequest> resourceRequests) {
+ // Update future requirements
+ decResourceRequest(node.getNodeName(), nodeLocalRequest);
+
+ ResourceRequest rackLocalRequest = resourceRequestMap.get(
+ node.getRackName());
+ decResourceRequest(node.getRackName(), rackLocalRequest);
+
+ ResourceRequest offRackRequest = resourceRequestMap.get(
+ ResourceRequest.ANY);
+ decrementOutstanding(schedulerKey, offRackRequest);
+
+ // Update cloned NodeLocal, RackLocal and OffRack requests for recovery
+ resourceRequests.add(cloneResourceRequest(nodeLocalRequest));
+ resourceRequests.add(cloneResourceRequest(rackLocalRequest));
+ resourceRequests.add(cloneResourceRequest(offRackRequest));
+ }
+
+ private void decResourceRequest(String resourceName,
+ ResourceRequest request) {
+ request.setNumContainers(request.getNumContainers() - 1);
+ if (request.getNumContainers() == 0) {
+ resourceRequestMap.remove(resourceName);
+ }
+ }
+
+ @Override
+ public boolean canAllocate(NodeType type, SchedulerNode node) {
+ try {
+ readLock.lock();
+ ResourceRequest r = resourceRequestMap.get(
+ ResourceRequest.ANY);
+ if (r == null || r.getNumContainers() <= 0) {
+ return false;
+ }
+ if (type == NodeType.RACK_LOCAL || type == NodeType.NODE_LOCAL) {
+ r = resourceRequestMap.get(node.getRackName());
+ if (r == null || r.getNumContainers() <= 0) {
+ return false;
+ }
+ if (type == NodeType.NODE_LOCAL) {
+ r = resourceRequestMap.get(node.getNodeName());
+ if (r == null || r.getNumContainers() <= 0) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean canDelayTo(String resourceName) {
+ try {
+ readLock.lock();
+ ResourceRequest request = getResourceRequest(resourceName);
+ return request == null || request.getRelaxLocality();
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ @Override
+ public boolean acceptNodePartition(String nodePartition,
+ SchedulingMode schedulingMode) {
+ // We will only look at node label = nodeLabelToLookAt according to
+ // schedulingMode and partition of node.
+ String nodePartitionToLookAt;
+ if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
+ nodePartitionToLookAt = nodePartition;
+ } else {
+ nodePartitionToLookAt = RMNodeLabelsManager.NO_LABEL;
+ }
+
+ return primaryRequestedPartition.equals(nodePartitionToLookAt);
+ }
+
+ @Override
+ public String getPrimaryRequestedNodePartition() {
+ return primaryRequestedPartition;
+ }
+
+ @Override
+ public int getUniqueLocationAsks() {
+ return resourceRequestMap.size();
+ }
+
+ @Override
+ public void showRequests() {
+ for (ResourceRequest request : resourceRequestMap.values()) {
+ if (request.getNumContainers() > 0) {
+ LOG.debug("\tRequest=" + request);
+ }
+ }
+ }
+
+ @Override
+ public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
+ NodeType type, SchedulerNode node) {
+ try {
+ writeLock.lock();
+
+ List<ResourceRequest> resourceRequests = new ArrayList<>();
+
+ ResourceRequest request;
+ if (type == NodeType.NODE_LOCAL) {
+ request = resourceRequestMap.get(node.getNodeName());
+ } else if (type == NodeType.RACK_LOCAL) {
+ request = resourceRequestMap.get(node.getRackName());
+ } else{
+ request = resourceRequestMap.get(ResourceRequest.ANY);
+ }
+
+ if (type == NodeType.NODE_LOCAL) {
+ allocateNodeLocal(schedulerKey, node, request, resourceRequests);
+ } else if (type == NodeType.RACK_LOCAL) {
+ allocateRackLocal(schedulerKey, node, request, resourceRequests);
+ } else{
+ allocateOffSwitch(schedulerKey, request, resourceRequests);
+ }
+
+ return resourceRequests;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public Iterator<String> getAcceptedResouceNames() {
+ try {
+ readLock.lock();
+ return resourceRequestMap.keySet().iterator();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
deleted file mode 100644
index 6cc8cc7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalitySchedulingPlacementSet.java
+++ /dev/null
@@ -1,416 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
-
-import org.apache.commons.collections.IteratorUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
-import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class LocalitySchedulingPlacementSet<N extends SchedulerNode>
- implements SchedulingPlacementSet<N> {
- private static final Log LOG =
- LogFactory.getLog(LocalitySchedulingPlacementSet.class);
-
- private final Map<String, ResourceRequest> resourceRequestMap =
- new ConcurrentHashMap<>();
- private AppSchedulingInfo appSchedulingInfo;
- private volatile String primaryRequestedPartition =
- RMNodeLabelsManager.NO_LABEL;
-
- private final ReentrantReadWriteLock.ReadLock readLock;
- private final ReentrantReadWriteLock.WriteLock writeLock;
-
- public LocalitySchedulingPlacementSet(AppSchedulingInfo info) {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- readLock = lock.readLock();
- writeLock = lock.writeLock();
- this.appSchedulingInfo = info;
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public Iterator<N> getPreferredNodeIterator(
- PlacementSet<N> clusterPlacementSet) {
- // Now only handle the case that single node in placementSet
- // TODO, Add support to multi-hosts inside placement-set which is passed in.
-
- N singleNode = PlacementSetUtils.getSingleNode(clusterPlacementSet);
- if (null != singleNode) {
- return IteratorUtils.singletonIterator(singleNode);
- }
-
- return IteratorUtils.emptyIterator();
- }
-
- private boolean hasRequestLabelChanged(ResourceRequest requestOne,
- ResourceRequest requestTwo) {
- String requestOneLabelExp = requestOne.getNodeLabelExpression();
- String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
- // First request label expression can be null and second request
- // is not null then we have to consider it as changed.
- if ((null == requestOneLabelExp) && (null != requestTwoLabelExp)) {
- return true;
- }
- // If the label is not matching between both request when
- // requestOneLabelExp is not null.
- return ((null != requestOneLabelExp) && !(requestOneLabelExp
- .equals(requestTwoLabelExp)));
- }
-
- private void updateNodeLabels(ResourceRequest request) {
- String resourceName = request.getResourceName();
- if (resourceName.equals(ResourceRequest.ANY)) {
- ResourceRequest previousAnyRequest =
- getResourceRequest(resourceName);
-
- // When there is change in ANY request label expression, we should
- // update label for all resource requests already added of same
- // priority as ANY resource request.
- if ((null == previousAnyRequest) || hasRequestLabelChanged(
- previousAnyRequest, request)) {
- for (ResourceRequest r : resourceRequestMap.values()) {
- if (!r.getResourceName().equals(ResourceRequest.ANY)) {
- r.setNodeLabelExpression(request.getNodeLabelExpression());
- }
- }
- }
- } else{
- ResourceRequest anyRequest = getResourceRequest(ResourceRequest.ANY);
- if (anyRequest != null) {
- request.setNodeLabelExpression(anyRequest.getNodeLabelExpression());
- }
- }
- }
-
- @Override
- public ResourceRequestUpdateResult updateResourceRequests(
- Collection<ResourceRequest> requests,
- boolean recoverPreemptedRequestForAContainer) {
- try {
- this.writeLock.lock();
-
- ResourceRequestUpdateResult updateResult = null;
-
- // Update resource requests
- for (ResourceRequest request : requests) {
- String resourceName = request.getResourceName();
-
- // Update node labels if required
- updateNodeLabels(request);
-
- // Increment number of containers if recovering preempted resources
- ResourceRequest lastRequest = resourceRequestMap.get(resourceName);
- if (recoverPreemptedRequestForAContainer && lastRequest != null) {
- request.setNumContainers(lastRequest.getNumContainers() + 1);
- }
-
- // Update asks
- resourceRequestMap.put(resourceName, request);
-
- if (resourceName.equals(ResourceRequest.ANY)) {
- String partition = request.getNodeLabelExpression() == null ?
- RMNodeLabelsManager.NO_LABEL :
- request.getNodeLabelExpression();
-
- this.primaryRequestedPartition = partition;
-
- //update the applications requested labels set
- appSchedulingInfo.addRequestedPartition(partition);
-
- updateResult = new ResourceRequestUpdateResult(lastRequest, request);
- }
- }
- return updateResult;
- } finally {
- this.writeLock.unlock();
- }
- }
-
- @Override
- public Map<String, ResourceRequest> getResourceRequests() {
- return resourceRequestMap;
- }
-
- private ResourceRequest getResourceRequest(String resourceName) {
- return resourceRequestMap.get(resourceName);
- }
-
- @Override
- public PendingAsk getPendingAsk(String resourceName) {
- try {
- readLock.lock();
- ResourceRequest request = getResourceRequest(resourceName);
- if (null == request) {
- return PendingAsk.ZERO;
- } else{
- return new PendingAsk(request.getCapability(),
- request.getNumContainers());
- }
- } finally {
- readLock.unlock();
- }
-
- }
-
- @Override
- public int getOutstandingAsksCount(String resourceName) {
- try {
- readLock.lock();
- ResourceRequest request = getResourceRequest(resourceName);
- if (null == request) {
- return 0;
- } else{
- return request.getNumContainers();
- }
- } finally {
- readLock.unlock();
- }
-
- }
-
- private void decrementOutstanding(SchedulerRequestKey schedulerRequestKey,
- ResourceRequest offSwitchRequest) {
- int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
- offSwitchRequest.setNumContainers(numOffSwitchContainers);
-
- // Do we have any outstanding requests?
- // If there is nothing, we need to deactivate this application
- if (numOffSwitchContainers == 0) {
- appSchedulingInfo.getSchedulerKeys().remove(schedulerRequestKey);
- appSchedulingInfo.checkForDeactivation();
- resourceRequestMap.remove(ResourceRequest.ANY);
- if (resourceRequestMap.isEmpty()) {
- appSchedulingInfo.removePlacementSets(schedulerRequestKey);
- }
- }
-
- appSchedulingInfo.decPendingResource(
- offSwitchRequest.getNodeLabelExpression(),
- offSwitchRequest.getCapability());
- }
-
- public ResourceRequest cloneResourceRequest(ResourceRequest request) {
- ResourceRequest newRequest = ResourceRequest.newBuilder()
- .priority(request.getPriority())
- .allocationRequestId(request.getAllocationRequestId())
- .resourceName(request.getResourceName())
- .capability(request.getCapability())
- .numContainers(1)
- .relaxLocality(request.getRelaxLocality())
- .nodeLabelExpression(request.getNodeLabelExpression()).build();
- return newRequest;
- }
-
- /**
- * The {@link ResourceScheduler} is allocating data-local resources to the
- * application.
- */
- private void allocateRackLocal(SchedulerRequestKey schedulerKey,
- SchedulerNode node, ResourceRequest rackLocalRequest,
- List<ResourceRequest> resourceRequests) {
- // Update future requirements
- decResourceRequest(node.getRackName(), rackLocalRequest);
-
- ResourceRequest offRackRequest = resourceRequestMap.get(
- ResourceRequest.ANY);
- decrementOutstanding(schedulerKey, offRackRequest);
-
- // Update cloned RackLocal and OffRack requests for recovery
- resourceRequests.add(cloneResourceRequest(rackLocalRequest));
- resourceRequests.add(cloneResourceRequest(offRackRequest));
- }
-
- /**
- * The {@link ResourceScheduler} is allocating data-local resources to the
- * application.
- */
- private void allocateOffSwitch(SchedulerRequestKey schedulerKey,
- ResourceRequest offSwitchRequest,
- List<ResourceRequest> resourceRequests) {
- // Update future requirements
- decrementOutstanding(schedulerKey, offSwitchRequest);
- // Update cloned OffRack requests for recovery
- resourceRequests.add(cloneResourceRequest(offSwitchRequest));
- }
-
-
- /**
- * The {@link ResourceScheduler} is allocating data-local resources to the
- * application.
- */
- private void allocateNodeLocal(SchedulerRequestKey schedulerKey,
- SchedulerNode node, ResourceRequest nodeLocalRequest,
- List<ResourceRequest> resourceRequests) {
- // Update future requirements
- decResourceRequest(node.getNodeName(), nodeLocalRequest);
-
- ResourceRequest rackLocalRequest = resourceRequestMap.get(
- node.getRackName());
- decResourceRequest(node.getRackName(), rackLocalRequest);
-
- ResourceRequest offRackRequest = resourceRequestMap.get(
- ResourceRequest.ANY);
- decrementOutstanding(schedulerKey, offRackRequest);
-
- // Update cloned NodeLocal, RackLocal and OffRack requests for recovery
- resourceRequests.add(cloneResourceRequest(nodeLocalRequest));
- resourceRequests.add(cloneResourceRequest(rackLocalRequest));
- resourceRequests.add(cloneResourceRequest(offRackRequest));
- }
-
- private void decResourceRequest(String resourceName,
- ResourceRequest request) {
- request.setNumContainers(request.getNumContainers() - 1);
- if (request.getNumContainers() == 0) {
- resourceRequestMap.remove(resourceName);
- }
- }
-
- @Override
- public boolean canAllocate(NodeType type, SchedulerNode node) {
- try {
- readLock.lock();
- ResourceRequest r = resourceRequestMap.get(
- ResourceRequest.ANY);
- if (r == null || r.getNumContainers() <= 0) {
- return false;
- }
- if (type == NodeType.RACK_LOCAL || type == NodeType.NODE_LOCAL) {
- r = resourceRequestMap.get(node.getRackName());
- if (r == null || r.getNumContainers() <= 0) {
- return false;
- }
- if (type == NodeType.NODE_LOCAL) {
- r = resourceRequestMap.get(node.getNodeName());
- if (r == null || r.getNumContainers() <= 0) {
- return false;
- }
- }
- }
-
- return true;
- } finally {
- readLock.unlock();
- }
- }
-
- @Override
- public boolean canDelayTo(String resourceName) {
- try {
- readLock.lock();
- ResourceRequest request = getResourceRequest(resourceName);
- return request == null || request.getRelaxLocality();
- } finally {
- readLock.unlock();
- }
-
- }
-
- @Override
- public boolean acceptNodePartition(String nodePartition,
- SchedulingMode schedulingMode) {
- // We will only look at node label = nodeLabelToLookAt according to
- // schedulingMode and partition of node.
- String nodePartitionToLookAt;
- if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
- nodePartitionToLookAt = nodePartition;
- } else {
- nodePartitionToLookAt = RMNodeLabelsManager.NO_LABEL;
- }
-
- return primaryRequestedPartition.equals(nodePartitionToLookAt);
- }
-
- @Override
- public String getPrimaryRequestedNodePartition() {
- return primaryRequestedPartition;
- }
-
- @Override
- public int getUniqueLocationAsks() {
- return resourceRequestMap.size();
- }
-
- @Override
- public void showRequests() {
- for (ResourceRequest request : resourceRequestMap.values()) {
- if (request.getNumContainers() > 0) {
- LOG.debug("\tRequest=" + request);
- }
- }
- }
-
- @Override
- public List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
- NodeType type, SchedulerNode node) {
- try {
- writeLock.lock();
-
- List<ResourceRequest> resourceRequests = new ArrayList<>();
-
- ResourceRequest request;
- if (type == NodeType.NODE_LOCAL) {
- request = resourceRequestMap.get(node.getNodeName());
- } else if (type == NodeType.RACK_LOCAL) {
- request = resourceRequestMap.get(node.getRackName());
- } else{
- request = resourceRequestMap.get(ResourceRequest.ANY);
- }
-
- if (type == NodeType.NODE_LOCAL) {
- allocateNodeLocal(schedulerKey, node, request, resourceRequests);
- } else if (type == NodeType.RACK_LOCAL) {
- allocateRackLocal(schedulerKey, node, request, resourceRequests);
- } else{
- allocateOffSwitch(schedulerKey, request, resourceRequests);
- }
-
- return resourceRequests;
- } finally {
- writeLock.unlock();
- }
- }
-
- @Override
- public Iterator<String> getAcceptedResouceNames() {
- try {
- readLock.lock();
- return resourceRequestMap.keySet().iterator();
- } finally {
- readLock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSet.java
deleted file mode 100644
index 2e6c3ca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSet.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * <p>
- * PlacementSet is the central place that decide the order of node to fit
- * asks by application.
- * </p>
- *
- * <p>
- * Also, PlacementSet can cache results (for example, ordered list) for
- * better performance.
- * </p>
- *
- * <p>
- * PlacementSet can depend on one or more other PlacementSets.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface PlacementSet<N extends SchedulerNode> {
- /**
- * Get all nodes for this PlacementSet
- * @return all nodes for this PlacementSet
- */
- Map<NodeId, N> getAllNodes();
-
- /**
- * Version of the PlacementSet, can help other PlacementSet with dependencies
- * deciding if update is required
- * @return version
- */
- long getVersion();
-
- /**
- * Partition of the PlacementSet.
- * @return node partition
- */
- String getPartition();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSetUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSetUtils.java
deleted file mode 100644
index 405122b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/PlacementSetUtils.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-
-public class PlacementSetUtils {
- /*
- * If the {@link PlacementSet} only has one entry, return it. otherwise
- * return null
- */
- public static <N extends SchedulerNode> N getSingleNode(PlacementSet<N> ps) {
- N node = null;
- if (1 == ps.getAllNodes().size()) {
- node = ps.getAllNodes().values().iterator().next();
- }
-
- return node;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
deleted file mode 100644
index 3e0620e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SchedulingPlacementSet.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
-
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
-import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * <p>
- * Comparing to {@link PlacementSet}, this also maintains
- * pending ResourceRequests:
- * - When new ResourceRequest(s) added to scheduler, or,
- * - Or new container allocated, scheduler can notify corresponding
- * PlacementSet.
- * </p>
- *
- * <p>
- * Different set of resource requests (E.g., resource requests with the
- * same schedulerKey) can have one instance of PlacementSet, each PlacementSet
- * can have different ways to order nodes depends on requests.
- * </p>
- */
-public interface SchedulingPlacementSet<N extends SchedulerNode> {
- /**
- * Get iterator of preferred node depends on requirement and/or availability
- * @param clusterPlacementSet input cluster PlacementSet
- * @return iterator of preferred node
- */
- Iterator<N> getPreferredNodeIterator(PlacementSet<N> clusterPlacementSet);
-
- /**
- * Replace existing ResourceRequest by the new requests
- *
- * @param requests new ResourceRequests
- * @param recoverPreemptedRequestForAContainer if we're recovering resource
- * requests for preempted container
- * @return true if total pending resource changed
- */
- ResourceRequestUpdateResult updateResourceRequests(
- Collection<ResourceRequest> requests,
- boolean recoverPreemptedRequestForAContainer);
-
- /**
- * Get pending ResourceRequests by given schedulerRequestKey
- * @return Map of resourceName to ResourceRequest
- */
- Map<String, ResourceRequest> getResourceRequests();
-
- /**
- * Get pending ask for given resourceName. If there's no such pendingAsk,
- * returns {@link PendingAsk#ZERO}
- *
- * @param resourceName resourceName
- * @return PendingAsk
- */
- PendingAsk getPendingAsk(String resourceName);
-
- /**
- * Get #pending-allocations for given resourceName. If there's no such
- * pendingAsk, returns 0
- *
- * @param resourceName resourceName
- * @return #pending-allocations
- */
- int getOutstandingAsksCount(String resourceName);
-
- /**
- * Notify container allocated.
- * @param schedulerKey SchedulerRequestKey for this ResourceRequest
- * @param type Type of the allocation
- * @param node Which node this container allocated on
- * @return list of ResourceRequests deducted
- */
- List<ResourceRequest> allocate(SchedulerRequestKey schedulerKey,
- NodeType type, SchedulerNode node);
-
- /**
- * Returns list of accepted resourceNames.
- * @return Iterator of accepted resourceNames
- */
- Iterator<String> getAcceptedResouceNames();
-
- /**
- * We can still have pending requirement for a given NodeType and node
- * @param type Locality Type
- * @param node which node we will allocate on
- * @return true if we has pending requirement
- */
- boolean canAllocate(NodeType type, SchedulerNode node);
-
- /**
- * Can delay to give locality?
- * TODO (wangda): This should be moved out of SchedulingPlacementSet
- * and should belong to specific delay scheduling policy impl.
- *
- * @param resourceName resourceName
- * @return can/cannot
- */
- boolean canDelayTo(String resourceName);
-
- /**
- * Does this {@link SchedulingPlacementSet} accept resources on nodePartition?
- *
- * @param nodePartition nodePartition
- * @param schedulingMode schedulingMode
- * @return accepted/not
- */
- boolean acceptNodePartition(String nodePartition,
- SchedulingMode schedulingMode);
-
- /**
- * It is possible that one request can accept multiple node partition,
- * So this method returns primary node partition for pending resource /
- * headroom calculation.
- *
- * @return primary requested node partition
- */
- String getPrimaryRequestedNodePartition();
-
- /**
- * @return number of unique location asks with #pending greater than 0,
- * (like /rack1, host1, etc.).
- *
- * TODO (wangda): This should be moved out of SchedulingPlacementSet
- * and should belong to specific delay scheduling policy impl.
- */
- int getUniqueLocationAsks();
-
- /**
- * Print human-readable requests to LOG debug.
- */
- void showRequests();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimpleCandidateNodeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimpleCandidateNodeSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimpleCandidateNodeSet.java
new file mode 100644
index 0000000..31a2170
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimpleCandidateNodeSet.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * A simple CandidateNodeSet which keeps an unordered map
+ */
+public class SimpleCandidateNodeSet<N extends SchedulerNode>
+ implements CandidateNodeSet<N> {
+
+ private Map<NodeId, N> map;
+ private String partition;
+
+ public SimpleCandidateNodeSet(N node) {
+ if (null != node) {
+ // Only one node in the initial CandidateNodeSet
+ this.map = ImmutableMap.of(node.getNodeID(), node);
+ this.partition = node.getPartition();
+ } else {
+ this.map = Collections.emptyMap();
+ this.partition = NodeLabel.DEFAULT_NODE_LABEL_PARTITION;
+ }
+ }
+
+ public SimpleCandidateNodeSet(Map<NodeId, N> map, String partition) {
+ this.map = map;
+ this.partition = partition;
+ }
+
+ @Override
+ public Map<NodeId, N> getAllNodes() {
+ return map;
+ }
+
+ @Override
+ public long getVersion() {
+ return 0L;
+ }
+
+ @Override
+ public String getPartition() {
+ return partition;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java
deleted file mode 100644
index 48efaa1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SimplePlacementSet.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * A simple PlacementSet which keeps an unordered map
- */
-public class SimplePlacementSet<N extends SchedulerNode>
- implements PlacementSet<N> {
-
- private Map<NodeId, N> map;
- private String partition;
-
- public SimplePlacementSet(N node) {
- if (null != node) {
- // Only one node in the initial PlacementSet
- this.map = ImmutableMap.of(node.getNodeID(), node);
- this.partition = node.getPartition();
- } else {
- this.map = Collections.emptyMap();
- this.partition = NodeLabel.DEFAULT_NODE_LABEL_PARTITION;
- }
- }
-
- public SimplePlacementSet(Map<NodeId, N> map, String partition) {
- this.map = map;
- this.partition = partition;
- }
-
- @Override
- public Map<NodeId, N> getAllNodes() {
- return map;
- }
-
- @Override
- public long getVersion() {
- return 0L;
- }
-
- @Override
- public String getPartition() {
- return partition;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/package-info.java
new file mode 100644
index 0000000..e8268f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
+ * contains classes related to application monitor.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 1dea4ee..800d023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -140,7 +140,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSc
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@@ -4199,7 +4199,8 @@ public class TestCapacityScheduler {
scheduler.handle(new NodeRemovedSchedulerEvent(
rm.getRMContext().getRMNodes().get(nm2.getNodeId())));
// schedulerNode is removed, try allocate a container
- scheduler.allocateContainersToNode(new SimplePlacementSet<>(node), true);
+ scheduler.allocateContainersToNode(new SimpleCandidateNodeSet<>(node),
+ true);
AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
new AppAttemptRemovedSchedulerEvent(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index e34665d..0fcc86d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -147,9 +147,9 @@ public class TestChildQueueOrder {
// Next call - nothing
if (allocation > 0) {
doReturn(new CSAssignment(Resources.none(), type)).
- when(queue)
- .assignContainers(eq(clusterResource), any(PlacementSet.class),
- any(ResourceLimits.class), any(SchedulingMode.class));
+ when(queue).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
+ any(SchedulingMode.class));
// Mock the node's resource availability
Resource available = node.getUnallocatedResource();
@@ -159,9 +159,9 @@ public class TestChildQueueOrder {
return new CSAssignment(allocatedResource, type);
}
- }).
- when(queue).assignContainers(eq(clusterResource), any(PlacementSet.class),
- any(ResourceLimits.class), any(SchedulingMode.class));
+ }).when(queue).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
+ any(SchedulingMode.class));
doNothing().when(node).releaseContainer(any(ContainerId.class),
anyBoolean());
}
@@ -425,10 +425,10 @@ public class TestChildQueueOrder {
clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
InOrder allocationOrder = inOrder(d,b);
allocationOrder.verify(d).assignContainers(eq(clusterResource),
- any(PlacementSet.class), any(ResourceLimits.class),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
any(SchedulingMode.class));
allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), any(ResourceLimits.class),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
any(SchedulingMode.class));
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 2*GB, clusterResource);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 541539d..eacbf6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -60,7 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
@@ -88,13 +88,14 @@ public class TestContainerResizing {
@Override
public CSAssignment allocateContainersToNode(
- PlacementSet<FiCaSchedulerNode> ps, boolean withNodeHeartbeat) {
+ CandidateNodeSet<FiCaSchedulerNode> candidates,
+ boolean withNodeHeartbeat) {
try {
Thread.sleep(1000);
} catch(InterruptedException e) {
LOG.debug("Thread interrupted.");
}
- return super.allocateContainersToNode(ps, withNodeHeartbeat);
+ return super.allocateContainersToNode(candidates, withNodeHeartbeat);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 740ef33..71fddfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -656,7 +656,7 @@ public class TestNodeLabelContainerAllocation {
if (key.getPriority().getPriority() == priority) {
Assert.assertEquals("Expected partition is " + expectedPartition,
expectedPartition,
- info.getSchedulingPlacementSet(key)
+ info.getAppPlacementAllocator(key)
.getPrimaryRequestedNodePartition());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac4d2b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index cdbbc51..a9196d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -29,7 +29,6 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.when;
import java.util.HashMap;
@@ -54,7 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -181,8 +180,9 @@ public class TestParentQueue {
// Next call - nothing
if (allocation > 0) {
doReturn(new CSAssignment(Resources.none(), type)).when(queue)
- .assignContainers(eq(clusterResource), any(PlacementSet.class),
- any(ResourceLimits.class), any(SchedulingMode.class));
+ .assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
+ any(SchedulingMode.class));
// Mock the node's resource availability
Resource available = node.getUnallocatedResource();
@@ -192,8 +192,9 @@ public class TestParentQueue {
return new CSAssignment(allocatedResource, type);
}
- }).when(queue).assignContainers(eq(clusterResource), any(PlacementSet.class),
- any(ResourceLimits.class), any(SchedulingMode.class));
+ }).when(queue).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), any(ResourceLimits.class),
+ any(SchedulingMode.class));
}
private float computeQueueAbsoluteUsedCapacity(CSQueue queue,
@@ -274,13 +275,14 @@ public class TestParentQueue {
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
InOrder allocationOrder = inOrder(a, b);
allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(),
+ any(CandidateNodeSet.class), anyResourceLimits(),
any(SchedulingMode.class));
root.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
- allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 2*GB, clusterResource);
@@ -293,10 +295,12 @@ public class TestParentQueue {
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(b, a);
- allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 4*GB, clusterResource);
@@ -307,10 +311,12 @@ public class TestParentQueue {
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(b, a);
- allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 8*GB, clusterResource);
@@ -325,10 +331,12 @@ public class TestParentQueue {
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(a, b);
- allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 4*GB, clusterResource);
verifyQueueMetrics(b, 9*GB, clusterResource);
}
@@ -547,22 +555,25 @@ public class TestParentQueue {
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
InOrder allocationOrder = inOrder(a, c, b);
- allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- applyAllocationToQueue(clusterResource, 1*GB, a);
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ applyAllocationToQueue(clusterResource, 1 * GB, a);
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
- allocationOrder.verify(c).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- applyAllocationToQueue(clusterResource, 2*GB, root);
+ allocationOrder.verify(c).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ applyAllocationToQueue(clusterResource, 2 * GB, root);
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
applyAllocationToQueue(clusterResource, 2*GB, b);
verifyQueueMetrics(a, 1*GB, clusterResource);
verifyQueueMetrics(b, 6*GB, clusterResource);
@@ -586,24 +597,28 @@ public class TestParentQueue {
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(a, a2, a1, b, c);
- allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(a2).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(a).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(a2).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
applyAllocationToQueue(clusterResource, 2*GB, a);
root.assignContainers(clusterResource, node_2,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
- allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
applyAllocationToQueue(clusterResource, 2*GB, b);
root.assignContainers(clusterResource, node_2,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
- allocationOrder.verify(c).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(c).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 3*GB, clusterResource);
verifyQueueMetrics(b, 8*GB, clusterResource);
verifyQueueMetrics(c, 4*GB, clusterResource);
@@ -720,12 +735,14 @@ public class TestParentQueue {
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
InOrder allocationOrder = inOrder(a);
allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
root.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(b);
allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 2*GB, clusterResource);
@@ -738,9 +755,11 @@ public class TestParentQueue {
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(b, a);
allocationOrder.verify(b).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
allocationOrder.verify(a).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(a, 2*GB, clusterResource);
verifyQueueMetrics(b, 4*GB, clusterResource);
@@ -800,10 +819,12 @@ public class TestParentQueue {
root.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
InOrder allocationOrder = inOrder(b2, b3);
- allocationOrder.verify(b2).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(b3).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b2).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(b3).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(b2, 1*GB, clusterResource);
verifyQueueMetrics(b3, 2*GB, clusterResource);
@@ -815,10 +836,12 @@ public class TestParentQueue {
root.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
allocationOrder = inOrder(b3, b2);
- allocationOrder.verify(b3).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
- allocationOrder.verify(b2).assignContainers(eq(clusterResource),
- any(PlacementSet.class), anyResourceLimits(), any(SchedulingMode.class));
+ allocationOrder.verify(b3).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
+ allocationOrder.verify(b2).assignContainers(eq(clusterResource),
+ any(CandidateNodeSet.class), anyResourceLimits(),
+ any(SchedulingMode.class));
verifyQueueMetrics(b2, 1*GB, clusterResource);
verifyQueueMetrics(b3, 3*GB, clusterResource);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDFS-12791. NameNode Fsck http
Connection can timeout for directories with multiple levels. Contributed by
Mukul Kumar Singh.
Posted by su...@apache.org.
HDFS-12791. NameNode Fsck http Connection can timeout for directories with multiple levels. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10a1f557
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10a1f557
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10a1f557
Branch: refs/heads/YARN-5881
Commit: 10a1f557e775e7a55958b106dd10021ac7394843
Parents: 5eb7dbe
Author: Chen Liang <cl...@apache.org>
Authored: Thu Nov 9 18:47:34 2017 -0800
Committer: Chen Liang <cl...@apache.org>
Committed: Thu Nov 9 18:47:34 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/NamenodeFsck.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10a1f557/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 5872955..b6d6971 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -471,6 +471,13 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
void check(String parent, HdfsFileStatus file, Result replRes, Result ecRes)
throws IOException {
String path = file.getFullName(parent);
+ if (showprogress &&
+ (totalDirs + totalSymlinks + replRes.totalFiles + ecRes.totalFiles)
+ % 100 == 0) {
+ out.println();
+ out.flush();
+ }
+
if (file.isDirectory()) {
checkDir(path, replRes, ecRes);
return;
@@ -489,10 +496,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
final Result r = file.getErasureCodingPolicy() != null ? ecRes: replRes;
collectFileSummary(path, file, r, blocks);
- if (showprogress && (replRes.totalFiles + ecRes.totalFiles) % 100 == 0) {
- out.println();
- out.flush();
- }
collectBlocksSummary(parent, file, r, blocks);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: MAPREDUCE-7001. Moving logging
APIs over to slf4j in hadoop-mapreduce-client-shuffle. Contributed by
Jinjiang Ling.
Posted by su...@apache.org.
MAPREDUCE-7001. Moving logging APIs over to slf4j in hadoop-mapreduce-client-shuffle. Contributed by Jinjiang Ling.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4c220ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4c220ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4c220ee
Branch: refs/heads/YARN-5881
Commit: e4c220ee4fdc4550275bda0fa9468d7d87d143c3
Parents: ffee10b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Nov 8 19:28:08 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Nov 8 19:28:08 2017 +0900
----------------------------------------------------------------------
.../apache/hadoop/mapred/FadvisedChunkedFile.java | 7 ++++---
.../apache/hadoop/mapred/FadvisedFileRegion.java | 7 ++++---
.../org/apache/hadoop/mapred/ShuffleHandler.java | 13 +++++++------
.../hadoop/mapred/TestFadvisedFileRegion.java | 18 +++++++++---------
.../apache/hadoop/mapred/TestShuffleHandler.java | 7 ++++---
5 files changed, 28 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c220ee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
index 7e24e89..6a4e3b4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java
@@ -22,11 +22,11 @@ import java.io.FileDescriptor;
import java.io.IOException;
import java.io.RandomAccessFile;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
@@ -34,7 +34,8 @@ import org.jboss.netty.handler.stream.ChunkedFile;
public class FadvisedChunkedFile extends ChunkedFile {
- private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FadvisedChunkedFile.class);
private final boolean manageOsCache;
private final int readaheadLength;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c220ee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
index 79045f9..4b2c8cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
@@ -25,11 +25,11 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.ReadaheadPool;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
@@ -39,7 +39,8 @@ import com.google.common.annotations.VisibleForTesting;
public class FadvisedFileRegion extends DefaultFileRegion {
- private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FadvisedFileRegion.class);
private final boolean manageOsCache;
private final int readaheadLength;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c220ee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index bf8241d..ec992fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -54,8 +54,6 @@ import java.util.regex.Pattern;
import javax.crypto.SecretKey;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputByteBuffer;
@@ -134,6 +132,7 @@ import org.jboss.netty.util.CharsetUtil;
import org.jboss.netty.util.HashedWheelTimer;
import org.jboss.netty.util.Timer;
import org.eclipse.jetty.http.HttpHeader;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
@@ -148,9 +147,10 @@ import com.google.protobuf.ByteString;
public class ShuffleHandler extends AuxiliaryService {
- private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
- private static final Log AUDITLOG =
- LogFactory.getLog(ShuffleHandler.class.getName()+".audit");
+ private static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(ShuffleHandler.class);
+ private static final org.slf4j.Logger AUDITLOG =
+ LoggerFactory.getLogger(ShuffleHandler.class.getName()+".audit");
public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
@@ -775,7 +775,8 @@ public class ShuffleHandler extends AuxiliaryService {
}
private static class LevelDBLogger implements Logger {
- private static final Log LOG = LogFactory.getLog(LevelDBLogger.class);
+ private static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(LevelDBLogger.class);
@Override
public void log(String message) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c220ee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
index 13d260f..242382e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestFadvisedFileRegion.java
@@ -25,18 +25,18 @@ import java.io.RandomAccessFile;
import java.nio.channels.WritableByteChannel;
import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestFadvisedFileRegion {
private final int FILE_SIZE = 16*1024*1024;
- private static final Log LOG =
- LogFactory.getLog(TestFadvisedFileRegion.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestFadvisedFileRegion.class);
@Test(timeout = 100000)
public void testCustomShuffleTransfer() throws IOException {
@@ -66,7 +66,7 @@ public class TestFadvisedFileRegion {
try{
out.write(initBuff);
} finally {
- IOUtils.cleanup(LOG, out);
+ IOUtils.cleanupWithLogger(LOG, out);
}
@@ -106,9 +106,9 @@ public class TestFadvisedFileRegion {
if (fileRegion != null) {
fileRegion.releaseExternalResources();
}
- IOUtils.cleanup(LOG, target);
- IOUtils.cleanup(LOG, targetFile);
- IOUtils.cleanup(LOG, inputFile);
+ IOUtils.cleanupWithLogger(LOG, target);
+ IOUtils.cleanupWithLogger(LOG, targetFile);
+ IOUtils.cleanupWithLogger(LOG, inputFile);
}
//Read the target file and verify that copy is done correctly
@@ -123,7 +123,7 @@ public class TestFadvisedFileRegion {
Assert.assertEquals(initBuff[position+i], buff[i]);
}
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
//delete files and folders
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c220ee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 250dcf6..8017248 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -48,8 +48,6 @@ import java.util.Map;
import java.util.zip.CheckedOutputStream;
import java.util.zip.Checksum;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -99,10 +97,13 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.mockito.Mockito;
import org.eclipse.jetty.http.HttpHeader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestShuffleHandler {
static final long MiB = 1024 * 1024;
- private static final Log LOG = LogFactory.getLog(TestShuffleHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestShuffleHandler.class);
private static final File ABS_LOG_DIR = GenericTestUtils.getTestDir(
TestShuffleHandler.class.getSimpleName() + "LocDir");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-7371. Added allocateRequestId
in NMContainerStatus for recovery. Contributed by Chandni Singh
Posted by su...@apache.org.
YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d0738
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d0738
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d0738
Branch: refs/heads/YARN-5881
Commit: a55d0738f1eba225c45578bbdabd0e5a6d6f5392
Parents: a127f7b
Author: Jian He <ji...@apache.org>
Authored: Sat Nov 4 23:15:21 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 13:30:20 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceTestUtils.java | 10 +-
.../yarn/service/TestYarnNativeServices.java | 105 ++++++++++++++++---
.../yarn/security/ContainerTokenIdentifier.java | 20 +++-
.../src/main/proto/yarn_security_token.proto | 1 +
.../api/protocolrecords/NMContainerStatus.java | 18 +++-
.../impl/pb/NMContainerStatusPBImpl.java | 15 ++-
.../OpportunisticContainerAllocator.java | 2 +-
.../yarn_server_common_service_protos.proto | 1 +
.../containermanager/ContainerManagerImpl.java | 5 +-
.../container/ContainerImpl.java | 8 +-
.../scheduler/AbstractYarnScheduler.java | 1 +
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../security/RMContainerTokenSecretManager.java | 36 ++-----
.../server/resourcemanager/TestRMRestart.java | 2 +-
.../TestResourceTrackerService.java | 6 +-
.../capacity/TestContainerAllocation.java | 5 +-
16 files changed, 177 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 19a5177..cf32880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -131,12 +131,16 @@ public class ServiceTestUtils {
return fs;
}
+ protected MiniYARNCluster getYarnCluster() {
+ return yarnCluster;
+ }
+
protected void setupInternal(int numNodeManager)
throws Exception {
LOG.info("Starting up YARN cluster");
-// Logger rootLogger = LogManager.getRootLogger();
-// rootLogger.setLevel(Level.DEBUG);
- setConf(new YarnConfiguration());
+ if (conf == null) {
+ setConf(new YarnConfiguration());
+ }
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
// reduce the teardown waiting time
conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 64fcf57..f98d90a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,12 +18,13 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -41,17 +42,12 @@ import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@@ -67,7 +63,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
@Before
public void setup() throws Exception {
- setupInternal(NUM_NMS);
+ File tmpYarnDir = new File("target", "tmp");
+ FileUtils.deleteQuietly(tmpYarnDir);
}
@After
@@ -84,6 +81,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// 6. Destroy the service
@Test (timeout = 200000)
public void testCreateFlexStopDestroyService() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
@@ -135,6 +133,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
// Check containers for compa started before containers for compb
@Test (timeout = 200000)
public void testComponentStartOrder() throws Exception {
+ setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service exampleApp = new Service();
exampleApp.setName("teststartorder");
@@ -155,6 +154,70 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(exampleApp.getName());
}
+ // Test to verify recovery of SeviceMaster after RM is restarted.
+ // 1. Create an example service.
+ // 2. Restart RM.
+ // 3. Fail the application attempt.
+ // 4. Verify ServiceMaster recovers.
+ @Test(timeout = 200000)
+ public void testRecoverComponentsAfterRMRestart() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.setBoolean(
+ YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setLong(YarnConfiguration.NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ 500L);
+
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
+ setConf(conf);
+ setupInternal(NUM_NMS);
+
+ ServiceClient client = createClient();
+ Service exampleApp = createExampleApplication();
+ client.actionCreate(exampleApp);
+ waitForAllCompToBeReady(client, exampleApp);
+
+ LOG.info("Restart the resource manager");
+ getYarnCluster().restartResourceManager(
+ getYarnCluster().getActiveRMIndex());
+ GenericTestUtils.waitFor(() ->
+ getYarnCluster().getResourceManager().getServiceState() ==
+ org.apache.hadoop.service.Service.STATE.STARTED, 2000, 200000);
+ Assert.assertTrue("node managers connected",
+ getYarnCluster().waitForNodeManagersToConnect(5000));
+
+ ApplicationId exampleAppId = ApplicationId.fromString(exampleApp.getId());
+ ApplicationAttemptId applicationAttemptId = client.getYarnClient()
+ .getApplicationReport(exampleAppId).getCurrentApplicationAttemptId();
+
+ Multimap<String, String> containersBeforeFailure = getContainersForAllComp(
+ client, exampleApp);
+
+ LOG.info("Fail the application attempt {}", applicationAttemptId);
+ client.getYarnClient().failApplicationAttempt(applicationAttemptId);
+ //wait until attempt 2 is running
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ApplicationReport ar = client.getYarnClient()
+ .getApplicationReport(exampleAppId);
+ return ar.getCurrentApplicationAttemptId().getAttemptId() == 2 &&
+ ar.getYarnApplicationState() == YarnApplicationState.RUNNING;
+ } catch (YarnException | IOException e) {
+ throw new RuntimeException("while waiting", e);
+ }
+ }, 2000, 200000);
+
+ Multimap<String, String> containersAfterFailure = getContainersForAllComp(
+ client, exampleApp);
+ Assert.assertEquals("component container affected by restart",
+ containersBeforeFailure, containersAfterFailure);
+
+ LOG.info("Stop/destroy service {}", exampleApp);
+ client.actionStop(exampleApp.getName(), true);
+ client.actionDestroy(exampleApp.getName());
+ }
+
// Check containers launched are in dependency order
// Get all containers into a list and sort based on container launch time e.g.
// compa-c1, compa-c2, compb-c1, compb-c2;
@@ -297,6 +360,24 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}, 2000, 200000);
}
+ /**
+ * Get all containers of a service.
+ */
+ private Multimap<String, String> getContainersForAllComp(ServiceClient client,
+ Service example) throws IOException, YarnException {
+
+ Multimap<String, String> allContainers = HashMultimap.create();
+ Service retrievedApp = client.getStatus(example.getName());
+ retrievedApp.getComponents().forEach(component -> {
+ if (component.getContainers() != null) {
+ component.getContainers().forEach(container -> {
+ allContainers.put(component.getName(), container.getId());
+ });
+ }
+ });
+ return allContainers;
+ }
+
private ServiceClient createClient() throws Exception {
ServiceClient client = new ServiceClient() {
@Override protected Path addJarResource(String appName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 07917ef..9e7d132 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -115,7 +115,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
this(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp,
masterKeyId, rmIdentifier, priority, creationTime,
logAggregationContext, nodeLabelExpression, containerType,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
}
public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
@@ -123,6 +123,19 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
ContainerType containerType, ExecutionType executionType) {
+
+ this(containerID, containerVersion, hostName, appSubmitter, r,
+ expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime,
+ logAggregationContext, nodeLabelExpression, containerType,
+ executionType, -1);
+ }
+
+ public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
+ String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+ int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext, String nodeLabelExpression,
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
ContainerTokenIdentifierProto.Builder builder =
ContainerTokenIdentifierProto.newBuilder();
if (containerID != null) {
@@ -152,6 +165,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
}
builder.setContainerType(convertToProtoFormat(containerType));
builder.setExecutionType(convertToProtoFormat(executionType));
+ builder.setAllocationRequestId(allocationRequestId);
proto = builder.build();
}
@@ -243,6 +257,10 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
return new LogAggregationContextPBImpl(proto.getLogAggregationContext());
}
+ public long getAllocationRequestId() {
+ return proto.getAllocationRequestId();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
index 851920d..d8288ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
@@ -53,6 +53,7 @@ message ContainerTokenIdentifierProto {
optional ContainerTypeProto containerType = 12;
optional ExecutionTypeProto executionType = 13 [default = GUARANTEED];
optional int32 version = 14 [default = 0];
+ optional int64 allocation_request_id = 15 [default = -1];
}
message ClientToAMTokenIdentifierProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
index 180add8..1a095f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
@@ -41,14 +41,14 @@ public abstract class NMContainerStatus {
long creationTime) {
return newInstance(containerId, version, containerState, allocatedResource,
diagnostics, containerExitStatus, priority, creationTime,
- CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED);
+ CommonNodeLabelsManager.NO_LABEL, ExecutionType.GUARANTEED, -1);
}
public static NMContainerStatus newInstance(ContainerId containerId,
int version, ContainerState containerState, Resource allocatedResource,
String diagnostics, int containerExitStatus, Priority priority,
long creationTime, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, long allocationRequestId) {
NMContainerStatus status =
Records.newRecord(NMContainerStatus.class);
status.setContainerId(containerId);
@@ -61,6 +61,7 @@ public abstract class NMContainerStatus {
status.setCreationTime(creationTime);
status.setNodeLabelExpression(nodeLabelExpression);
status.setExecutionType(executionType);
+ status.setAllocationRequestId(allocationRequestId);
return status;
}
@@ -130,6 +131,19 @@ public abstract class NMContainerStatus {
public abstract void setNodeLabelExpression(
String nodeLabelExpression);
+ /**
+ * @return the <em>ID</em> corresponding to the original allocation request.
+ */
+ public abstract long getAllocationRequestId();
+
+ /**
+ * Set the <em>ID</em> corresponding to the original allocation request.
+ *
+ * @param allocationRequestId the <em>ID</em> corresponding to the original
+ * allocation request.
+ */
+ public abstract void setAllocationRequestId(long allocationRequestId);
+
public int getVersion() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
index 38df5f6..8ed02fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
@@ -91,7 +91,8 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
.append("Diagnostics: ").append(getDiagnostics()).append(", ")
.append("ExitStatus: ").append(getContainerExitStatus()).append(", ")
.append("NodeLabelExpression: ").append(getNodeLabelExpression())
- .append("Priority: ").append(getPriority())
+ .append("Priority: ").append(getPriority()).append(", ")
+ .append("AllocationRequestId: ").append(getAllocationRequestId())
.append("]");
return sb.toString();
}
@@ -270,6 +271,18 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
builder.setExecutionType(convertToProtoFormat(executionType));
}
+ @Override
+ public long getAllocationRequestId() {
+ NMContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestId) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestId);
+ }
+
private void mergeLocalToBuilder() {
if (this.containerId != null
&& !((ContainerIdPBImpl) containerId).getProto().equals(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index ede4958..1f53648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -564,7 +564,7 @@ public class OpportunisticContainerAllocator {
tokenSecretManager.getCurrentKey().getKeyId(), rmIdentifier,
schedulerKey.getPriority(), currTime,
null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, schedulerKey.getAllocationRequestId());
byte[] pwd =
tokenSecretManager.createPassword(containerTokenIdentifier);
Token containerToken = newContainerToken(node.getNodeId(), pwd,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 8e59f14..8c4fc69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -176,6 +176,7 @@ message NMContainerStatusProto {
optional string nodeLabelExpression = 8;
optional int32 version = 9;
optional ExecutionTypeProto executionType = 10 [default = GUARANTEED];
+ optional int64 allocation_request_id = 11 [default = -1];
}
message SCMUploaderNotifyRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 55119e0..831ba0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -437,7 +437,7 @@ public class ContainerManagerImpl extends CompositeService implements
throws IOException {
StartContainerRequest req = rcs.getStartRequest();
ContainerLaunchContext launchContext = req.getContainerLaunchContext();
- ContainerTokenIdentifier token = null;
+ ContainerTokenIdentifier token;
if(rcs.getCapability() != null) {
ContainerTokenIdentifier originalToken =
BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
@@ -449,7 +449,8 @@ public class ContainerManagerImpl extends CompositeService implements
originalToken.getCreationTime(),
originalToken.getLogAggregationContext(),
originalToken.getNodeLabelExpression(),
- originalToken.getContainerType(), originalToken.getExecutionType());
+ originalToken.getContainerType(), originalToken.getExecutionType(),
+ originalToken.getAllocationRequestId());
} else {
token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 17b24b4..901b013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -844,12 +844,14 @@ public class ContainerImpl implements Container {
public NMContainerStatus getNMContainerStatus() {
this.readLock.lock();
try {
- return NMContainerStatus.newInstance(this.containerId, this.version,
- getCurrentState(), getResource(), diagnostics.toString(), exitCode,
+ return NMContainerStatus.newInstance(this.containerId,
+ this.version, getCurrentState(), getResource(),
+ diagnostics.toString(), exitCode,
containerTokenIdentifier.getPriority(),
containerTokenIdentifier.getCreationTime(),
containerTokenIdentifier.getNodeLabelExpression(),
- containerTokenIdentifier.getExecutionType());
+ containerTokenIdentifier.getExecutionType(),
+ containerTokenIdentifier.getAllocationRequestId());
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 8ce6eb8..53f43e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -571,6 +571,7 @@ public abstract class AbstractYarnScheduler
status.getPriority(), null);
container.setVersion(status.getVersion());
container.setExecutionType(status.getExecutionType());
+ container.setAllocationRequestId(status.getAllocationRequestId());
ApplicationAttemptId attemptId =
container.getId().getApplicationAttemptId();
RMContainer rmContainer = new RMContainerImpl(container,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index db63cd8..ce71afa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -655,7 +655,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
container.getNodeId(), getUser(), container.getResource(),
container.getPriority(), rmContainer.getCreationTime(),
this.logAggregationContext, rmContainer.getNodeLabelExpression(),
- containerType, container.getExecutionType()));
+ containerType, container.getExecutionType(),
+ container.getAllocationRequestId()));
updateNMToken(container);
} catch (IllegalArgumentException e) {
// DNS might be down, skip returning this container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
index 677aa14..191900b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
@@ -167,7 +167,7 @@ public class RMContainerTokenSecretManager extends
}
/**
- * Helper function for creating ContainerTokens
+ * Helper function for creating ContainerTokens.
*
* @param containerId Container Id
* @param containerVersion Container Version
@@ -183,37 +183,13 @@ public class RMContainerTokenSecretManager extends
Resource capability, Priority priority, long createTime) {
return createContainerToken(containerId, containerVersion, nodeId,
appSubmitter, capability, priority, createTime,
- null, null, ContainerType.TASK);
+ null, null, ContainerType.TASK,
+ ExecutionType.GUARANTEED, -1);
}
/**
* Helper function for creating ContainerTokens.
*
- * @param containerId containerId.
- * @param containerVersion containerVersion.
- * @param nodeId nodeId.
- * @param appSubmitter appSubmitter.
- * @param capability capability.
- * @param priority priority.
- * @param createTime createTime.
- * @param logAggregationContext logAggregationContext.
- * @param nodeLabelExpression nodeLabelExpression.
- * @param containerType containerType.
- * @return the container-token.
- */
- public Token createContainerToken(ContainerId containerId,
- int containerVersion, NodeId nodeId, String appSubmitter,
- Resource capability, Priority priority, long createTime,
- LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType) {
- return createContainerToken(containerId, containerVersion, nodeId,
- appSubmitter, capability, priority, createTime, null, null,
- ContainerType.TASK, ExecutionType.GUARANTEED);
- }
-
- /**
- * Helper function for creating ContainerTokens
- *
* @param containerId Container Id
* @param containerVersion Container version
* @param nodeId Node Id
@@ -225,13 +201,15 @@ public class RMContainerTokenSecretManager extends
* @param nodeLabelExpression Node Label Expression
* @param containerType Container Type
* @param execType Execution Type
+ * @param allocationRequestId allocationRequestId
* @return the container-token
*/
public Token createContainerToken(ContainerId containerId,
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
- ContainerType containerType, ExecutionType execType) {
+ ContainerType containerType, ExecutionType execType,
+ long allocationRequestId) {
byte[] password;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp =
@@ -246,7 +224,7 @@ public class RMContainerTokenSecretManager extends
this.currentMasterKey.getMasterKey().getKeyId(),
ResourceManager.getClusterTimeStamp(), priority, createTime,
logAggregationContext, nodeLabelExpression, containerType,
- execType);
+ execType, allocationRequestId);
password = this.createPassword(tokenIdentifier);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index f0f51f3..a4f18ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2113,7 +2113,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
NMContainerStatus.newInstance(containerId, 0, containerState,
Resource.newInstance(1024, 1), "recover container", 0,
Priority.newInstance(0), 0, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
return containerReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 2bd745b..1cb2d0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -2088,17 +2088,17 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
NMContainerStatus.newInstance(c1, 1, ContainerState.RUNNING,
Resource.newInstance(1024, 1), "Dummy Queued OC",
ContainerExitStatus.INVALID, Priority.newInstance(5), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningOpp =
NMContainerStatus.newInstance(c2, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running OC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, -1);
NMContainerStatus runningGuar =
NMContainerStatus.newInstance(c3, 1, ContainerState.RUNNING,
Resource.newInstance(2048, 1), "Dummy Running GC",
ContainerExitStatus.INVALID, Priority.newInstance(6), 1234, "",
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, -1);
req.setContainerStatuses(Arrays.asList(queuedOpp, runningOpp, runningGuar));
// trying to register a invalid node.
RegisterNodeManagerResponse response =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b1ca72a..6f54d47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -295,12 +295,13 @@ public class TestContainerAllocation {
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExp,
- ContainerType containerType, ExecutionType executionType) {
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId) {
numRetries++;
return super.createContainerToken(containerId, containerVersion,
nodeId, appSubmitter, capability, priority, createTime,
logAggregationContext, nodeLabelExp, containerType,
- executionType);
+ executionType, allocationRequestId);
}
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-7060. Avoid taking locks when
sending heartbeats from the DataNode. Contributed by Jiandan Yang.
Posted by su...@apache.org.
HDFS-7060. Avoid taking locks when sending heartbeats from the DataNode. Contributed by Jiandan Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb8a6eea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb8a6eea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb8a6eea
Branch: refs/heads/YARN-5881
Commit: bb8a6eea52cb1e2c3d0b7f8b49a1bab9e4255acd
Parents: 51e882d
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Nov 8 10:22:13 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Nov 8 10:22:13 2017 +0800
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 51 ++++++++------------
.../datanode/fsdataset/impl/FsVolumeImpl.java | 40 +++++++--------
2 files changed, 37 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8a6eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 41c41e6..d4375cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -153,22 +153,22 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
public StorageReport[] getStorageReports(String bpid)
throws IOException {
List<StorageReport> reports;
- synchronized (statsLock) {
- List<FsVolumeImpl> curVolumes = volumes.getVolumes();
- reports = new ArrayList<>(curVolumes.size());
- for (FsVolumeImpl volume : curVolumes) {
- try (FsVolumeReference ref = volume.obtainReference()) {
- StorageReport sr = new StorageReport(volume.toDatanodeStorage(),
- false,
- volume.getCapacity(),
- volume.getDfsUsed(),
- volume.getAvailable(),
- volume.getBlockPoolUsed(bpid),
- volume.getNonDfsUsed());
- reports.add(sr);
- } catch (ClosedChannelException e) {
- continue;
- }
+ // Volumes are the references from a copy-on-write snapshot, so the
+ // access on the volume metrics doesn't require an additional lock.
+ List<FsVolumeImpl> curVolumes = volumes.getVolumes();
+ reports = new ArrayList<>(curVolumes.size());
+ for (FsVolumeImpl volume : curVolumes) {
+ try (FsVolumeReference ref = volume.obtainReference()) {
+ StorageReport sr = new StorageReport(volume.toDatanodeStorage(),
+ false,
+ volume.getCapacity(),
+ volume.getDfsUsed(),
+ volume.getAvailable(),
+ volume.getBlockPoolUsed(bpid),
+ volume.getNonDfsUsed());
+ reports.add(sr);
+ } catch (ClosedChannelException e) {
+ continue;
}
}
@@ -247,9 +247,6 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
private final int smallBufferSize;
- // Used for synchronizing access to usage stats
- private final Object statsLock = new Object();
-
final LocalFileSystem localFS;
private boolean blockPinningEnabled;
@@ -583,9 +580,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override // FSDatasetMBean
public long getDfsUsed() throws IOException {
- synchronized(statsLock) {
- return volumes.getDfsUsed();
- }
+ return volumes.getDfsUsed();
}
/**
@@ -593,9 +588,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override // FSDatasetMBean
public long getBlockPoolUsed(String bpid) throws IOException {
- synchronized(statsLock) {
- return volumes.getBlockPoolUsed(bpid);
- }
+ return volumes.getBlockPoolUsed(bpid);
}
/**
@@ -611,9 +604,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override // FSDatasetMBean
public long getCapacity() throws IOException {
- synchronized(statsLock) {
- return volumes.getCapacity();
- }
+ return volumes.getCapacity();
}
/**
@@ -621,9 +612,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
*/
@Override // FSDatasetMBean
public long getRemaining() throws IOException {
- synchronized(statsLock) {
- return volumes.getRemaining();
- }
+ return volumes.getRemaining();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8a6eea/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index b948fb7..7224e69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
import org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
-import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -342,43 +341,38 @@ public class FsVolumeImpl implements FsVolumeSpi {
private void decDfsUsedAndNumBlocks(String bpid, long value,
boolean blockFileDeleted) {
- try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
- BlockPoolSlice bp = bpSlices.get(bpid);
- if (bp != null) {
- bp.decDfsUsed(value);
- if (blockFileDeleted) {
- bp.decrNumBlocks();
- }
+ // BlockPoolSlice map is thread safe, and update the space used or
+ // number of blocks are atomic operations, so it doesn't require to
+ // hold the dataset lock.
+ BlockPoolSlice bp = bpSlices.get(bpid);
+ if (bp != null) {
+ bp.decDfsUsed(value);
+ if (blockFileDeleted) {
+ bp.decrNumBlocks();
}
}
}
void incDfsUsedAndNumBlocks(String bpid, long value) {
- try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
- BlockPoolSlice bp = bpSlices.get(bpid);
- if (bp != null) {
- bp.incDfsUsed(value);
- bp.incrNumBlocks();
- }
+ BlockPoolSlice bp = bpSlices.get(bpid);
+ if (bp != null) {
+ bp.incDfsUsed(value);
+ bp.incrNumBlocks();
}
}
void incDfsUsed(String bpid, long value) {
- try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
- BlockPoolSlice bp = bpSlices.get(bpid);
- if (bp != null) {
- bp.incDfsUsed(value);
- }
+ BlockPoolSlice bp = bpSlices.get(bpid);
+ if (bp != null) {
+ bp.incDfsUsed(value);
}
}
@VisibleForTesting
public long getDfsUsed() throws IOException {
long dfsUsed = 0;
- try(AutoCloseableLock lock = dataset.acquireDatasetLock()) {
- for(BlockPoolSlice s : bpSlices.values()) {
- dfsUsed += s.getDfsUsed();
- }
+ for(BlockPoolSlice s : bpSlices.values()) {
+ dfsUsed += s.getDfsUsed();
}
return dfsUsed;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-7343. Add a junit test for
ContainerScheduler recovery. (Sampada Dehankar via asuresh)
Posted by su...@apache.org.
YARN-7343. Add a junit test for ContainerScheduler recovery. (Sampada Dehankar via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb35a595
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb35a595
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb35a595
Branch: refs/heads/YARN-5881
Commit: cb35a59589f0b18e1989543f9e4c2a33c96d5ff7
Parents: a9c70b0
Author: Arun Suresh <as...@apache.org>
Authored: Wed Nov 8 08:14:02 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Nov 8 08:14:02 2017 -0800
----------------------------------------------------------------------
.../TestContainerSchedulerRecovery.java | 314 +++++++++++++++++++
1 file changed, 314 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb35a595/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
new file mode 100644
index 0000000..2ae8b97
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerRecovery.java
@@ -0,0 +1,314 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.doNothing;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests to verify that the {@link ContainerScheduler} is able to
+ * recover active containers based on RecoveredContainerStatus and
+ * ExecutionType.
+ */
+public class TestContainerSchedulerRecovery {
+
+ @Mock private NMContext context;
+
+ @Mock private NodeManagerMetrics metrics;
+
+ @Mock private AsyncDispatcher dispatcher;
+
+ @Mock private ContainerTokenIdentifier token;
+
+ @Mock private ContainerImpl container;
+
+ @Mock private ApplicationId appId;
+
+ @Mock private ApplicationAttemptId appAttemptId;
+
+ @Mock private ContainerId containerId;
+
+ @Mock private AllocationBasedResourceUtilizationTracker
+ allocationBasedResourceUtilizationTracker;
+
+ @InjectMocks private ContainerScheduler tempContainerScheduler =
+ new ContainerScheduler(context, dispatcher, metrics, 0);
+
+ private ContainerScheduler spy;
+
+ @Before public void setUp() throws Exception {
+ MockitoAnnotations.initMocks(this);
+ spy = spy(tempContainerScheduler);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
+ when(containerId.getApplicationAttemptId().getApplicationId())
+ .thenReturn(appId);
+ when(containerId.getContainerId()).thenReturn(123L);
+ doNothing().when(allocationBasedResourceUtilizationTracker)
+ .addContainerResources(container);
+ }
+
+ @After public void tearDown() {
+ }
+
+ /*Test if a container is recovered as QUEUED, GUARANTEED,
+ * it should be added to queuedGuaranteedContainers map.
+ * */
+ @Test public void testRecoverContainerQueuedGuaranteed()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(1, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as QUEUED, OPPORTUNISTIC,
+ * it should be added to queuedOpportunisticContainers map.
+ * */
+ @Test public void testRecoverContainerQueuedOpportunistic()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(1, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as PAUSED, GUARANTEED,
+ * it should be added to queuedGuaranteedContainers map.
+ * */
+ @Test public void testRecoverContainerPausedGuaranteed()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(1, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as PAUSED, OPPORTUNISTIC,
+ * it should be added to queuedOpportunisticContainers map.
+ * */
+ @Test public void testRecoverContainerPausedOpportunistic()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(1, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as LAUNCHED, GUARANTEED,
+ * it should be added to runningContainers map.
+ * */
+ @Test public void testRecoverContainerLaunchedGuaranteed()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.LAUNCHED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(1, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as LAUNCHED, OPPORTUNISTIC,
+ * it should be added to runningContainers map.
+ * */
+ @Test public void testRecoverContainerLaunchedOpportunistic()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.LAUNCHED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(1, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(1))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as REQUESTED, GUARANTEED,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testRecoverContainerRequestedGuaranteed()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.REQUESTED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as REQUESTED, OPPORTUNISTIC,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testRecoverContainerRequestedOpportunistic()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.REQUESTED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as COMPLETED, GUARANTEED,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testRecoverContainerCompletedGuaranteed()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.COMPLETED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.GUARANTEED);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as COMPLETED, OPPORTUNISTIC,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testRecoverContainerCompletedOpportunistic()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.COMPLETED;
+ when(token.getExecutionType()).thenReturn(ExecutionType.OPPORTUNISTIC);
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as GUARANTEED but no executionType set,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testContainerQueuedNoExecType()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.QUEUED;
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+
+ /*Test if a container is recovered as PAUSED but no executionType set,
+ * it should not be added to any map mentioned below.
+ * */
+ @Test public void testContainerPausedNoExecType()
+ throws IllegalArgumentException, IllegalAccessException {
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ RecoveredContainerStatus rcs = RecoveredContainerStatus.PAUSED;
+ when(container.getContainerTokenIdentifier()).thenReturn(token);
+ spy.recoverActiveContainer(container, rcs);
+ assertEquals(0, spy.getNumQueuedGuaranteedContainers());
+ assertEquals(0, spy.getNumQueuedOpportunisticContainers());
+ assertEquals(0, spy.getNumRunningContainers());
+ Mockito.verify(allocationBasedResourceUtilizationTracker, Mockito.times(0))
+ .addContainerResources(container);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org